* [PATCH v8 15/23] dma-direct: pass attrs to dma_capable() for DMA_ATTR_CC_SHARED checks
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jason Gunthorpe,
Jiri Pirko, Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
Teach dma_capable() about DMA_ATTR_CC_SHARED so the capability
check can reject encrypted DMA addresses for devices that require
unencrypted/shared DMA.
Also propagate DMA_ATTR_CC_SHARED in swiotlb_map() when the selected
SWIOTLB pool is decrypted so the capability check sees the correct DMA
address attribute.
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
arch/x86/kernel/amd_gart_64.c | 30 ++++++++++++++++--------------
drivers/xen/swiotlb-xen.c | 6 +++---
include/linux/dma-direct.h | 10 +++++++++-
kernel/dma/direct.h | 6 +++---
kernel/dma/swiotlb.c | 2 +-
5 files changed, 32 insertions(+), 22 deletions(-)
diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c
index e8000a56732e..b5f1f031d45b 100644
--- a/arch/x86/kernel/amd_gart_64.c
+++ b/arch/x86/kernel/amd_gart_64.c
@@ -180,22 +180,23 @@ static void iommu_full(struct device *dev, size_t size, int dir)
}
static inline int
-need_iommu(struct device *dev, unsigned long addr, size_t size)
+need_iommu(struct device *dev, unsigned long addr, size_t size, unsigned long attrs)
{
- return force_iommu || !dma_capable(dev, addr, size, true);
+ return force_iommu || !dma_capable(dev, addr, size, true, attrs);
}
static inline int
-nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
+nonforced_iommu(struct device *dev, unsigned long addr, size_t size,
+ unsigned long attrs)
{
- return !dma_capable(dev, addr, size, true);
+ return !dma_capable(dev, addr, size, true, attrs);
}
/* Map a single continuous physical area into the IOMMU.
* Caller needs to check if the iommu is needed and flush.
*/
static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
- size_t size, int dir, unsigned long align_mask)
+ size_t size, int dir, unsigned long align_mask, unsigned long attrs)
{
unsigned long npages = iommu_num_pages(phys_mem, size, PAGE_SIZE);
unsigned long iommu_page;
@@ -206,7 +207,7 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
iommu_page = alloc_iommu(dev, npages, align_mask);
if (iommu_page == -1) {
- if (!nonforced_iommu(dev, phys_mem, size))
+ if (!nonforced_iommu(dev, phys_mem, size, attrs))
return phys_mem;
if (panic_on_overflow)
panic("dma_map_area overflow %lu bytes\n", size);
@@ -231,10 +232,10 @@ static dma_addr_t gart_map_phys(struct device *dev, phys_addr_t paddr,
if (unlikely(attrs & DMA_ATTR_MMIO))
return DMA_MAPPING_ERROR;
- if (!need_iommu(dev, paddr, size))
+ if (!need_iommu(dev, paddr, size, attrs))
return paddr;
- bus = dma_map_area(dev, paddr, size, dir, 0);
+ bus = dma_map_area(dev, paddr, size, dir, 0, attrs);
flush_gart();
return bus;
@@ -289,7 +290,7 @@ static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
/* Fallback for dma_map_sg in case of overflow */
static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
- int nents, int dir)
+ int nents, int dir, unsigned long attrs)
{
struct scatterlist *s;
int i;
@@ -301,8 +302,8 @@ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
for_each_sg(sg, s, nents, i) {
unsigned long addr = sg_phys(s);
- if (nonforced_iommu(dev, addr, s->length)) {
- addr = dma_map_area(dev, addr, s->length, dir, 0);
+ if (nonforced_iommu(dev, addr, s->length, attrs)) {
+ addr = dma_map_area(dev, addr, s->length, dir, 0, attrs);
if (addr == DMA_MAPPING_ERROR) {
if (i > 0)
gart_unmap_sg(dev, sg, i, dir, 0);
@@ -401,7 +402,7 @@ static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
s->dma_address = addr;
BUG_ON(s->length == 0);
- nextneed = need_iommu(dev, addr, s->length);
+ nextneed = need_iommu(dev, addr, s->length, attrs);
/* Handle the previous not yet processed entries */
if (i > start) {
@@ -449,7 +450,7 @@ static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
/* When it was forced or merged try again in a dumb way */
if (force_iommu || iommu_merge) {
- out = dma_map_sg_nonforce(dev, sg, nents, dir);
+ out = dma_map_sg_nonforce(dev, sg, nents, dir, attrs);
if (out > 0)
return out;
}
@@ -473,7 +474,8 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
return vaddr;
*dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size,
- DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);
+ DMA_BIDIRECTIONAL,
+ (1UL << get_order(size)) - 1, attrs);
flush_gart();
if (unlikely(*dma_addr == DMA_MAPPING_ERROR))
goto out_free;
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 8c4abe65cd49..e2538824ef52 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -212,7 +212,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
BUG_ON(dir == DMA_NONE);
if (attrs & DMA_ATTR_MMIO) {
- if (unlikely(!dma_capable(dev, phys, size, false))) {
+ if (unlikely(!dma_capable(dev, phys, size, false, attrs))) {
dev_err_once(
dev,
"DMA addr %pa+%zu overflow (mask %llx, bus limit %llx).\n",
@@ -231,7 +231,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
* we can safely return the device addr and not worry about bounce
* buffering it.
*/
- if (dma_capable(dev, dev_addr, size, true) &&
+ if (dma_capable(dev, dev_addr, size, true, attrs) &&
!dma_kmalloc_needs_bounce(dev, size, dir) &&
!range_straddles_page_boundary(phys, size) &&
!xen_arch_need_swiotlb(dev, phys, dev_addr) &&
@@ -253,7 +253,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
/*
* Ensure that the address returned is DMA'ble
*/
- if (unlikely(!dma_capable(dev, dev_addr, size, true))) {
+ if (unlikely(!dma_capable(dev, dev_addr, size, true, attrs))) {
__swiotlb_tbl_unmap_single(dev, map, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC,
swiotlb_find_pool(dev, map));
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index 94fad4e7c11e..daa31a1adf7b 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -135,12 +135,20 @@ static inline bool force_dma_unencrypted(struct device *dev)
#endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */
static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size,
- bool is_ram)
+ bool is_ram, unsigned long attrs)
{
dma_addr_t end = addr + size - 1;
if (addr == DMA_MAPPING_ERROR)
return false;
+ /*
+ * The DMA address was derived from encrypted RAM, but this device
+ * requires unencrypted DMA addresses. Treat it as not DMA-capable
+ * so the caller can fall back to a suitable SWIOTLB pool.
+ */
+ if (!(attrs & DMA_ATTR_CC_SHARED) && force_dma_unencrypted(dev))
+ return false;
+
if (is_ram && !IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) &&
min(addr, end) < phys_to_dma(dev, PFN_PHYS(min_low_pfn)))
return false;
diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h
index 7140c208c123..e05dc7649366 100644
--- a/kernel/dma/direct.h
+++ b/kernel/dma/direct.h
@@ -101,15 +101,15 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev,
if (attrs & DMA_ATTR_MMIO) {
dma_addr = phys;
- if (unlikely(!dma_capable(dev, dma_addr, size, false)))
+ if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs)))
goto err_overflow;
} else if (attrs & DMA_ATTR_CC_SHARED) {
dma_addr = phys_to_dma_unencrypted(dev, phys);
- if (unlikely(!dma_capable(dev, dma_addr, size, false)))
+ if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs)))
goto err_overflow;
} else {
dma_addr = phys_to_dma(dev, phys);
- if (unlikely(!dma_capable(dev, dma_addr, size, true)) ||
+ if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) ||
dma_kmalloc_needs_bounce(dev, size, dir)) {
if (is_swiotlb_active(dev) &&
!(attrs & DMA_ATTR_REQUIRE_COHERENT))
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 335e27bc1e1f..b5960c2e98d8 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1697,7 +1697,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
else
dma_addr = phys_to_dma_encrypted(dev, swiotlb_addr);
- if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
+ if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs))) {
__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC,
swiotlb_find_pool(dev, swiotlb_addr));
--
2.43.0
^ permalink raw reply related
* [PATCH v8 14/23] dma-mapping: make dma_pgprot() honor __DMA_ATTR_ALLOC_CC_SHARED
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jason Gunthorpe,
Jiri Pirko, Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
Fold encrypted/decrypted pgprot selection into dma_pgprot() so callers
do not need to adjust the page protection separately.
Update dma_pgprot() to apply pgprot_decrypted() when DMA_ATTR_CC_SHARED or
__DMA_ATTR_ALLOC_CC_SHARED is set and pgprot_encrypted() otherwise Convert
the dma-direct mmap paths to pass DMA_ATTR_CC_SHARED instead of open-coding
force_dma_unencrypted() handling around dma_pgprot().
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 8 +++-----
kernel/dma/mapping.c | 16 ++++++++++++----
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 9f013ff8e0f6..f54c7929b97f 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -286,9 +286,6 @@ void *dma_direct_alloc(struct device *dev, size_t size,
if (remap) {
pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
- if (force_dma_unencrypted(dev))
- prot = pgprot_decrypted(prot);
-
/* remove any dirty cache lines on the kernel alias */
arch_dma_prep_coherent(page, size);
@@ -607,9 +604,10 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr));
int ret = -ENXIO;
- vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
if (force_dma_unencrypted(dev))
- vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
+ attrs |= DMA_ATTR_CC_SHARED;
+
+ vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
return ret;
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index d2f70b6ccd0f..a628820fd10e 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -537,13 +537,21 @@ EXPORT_SYMBOL(dma_get_sgtable_attrs);
*/
pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs)
{
+ pgprot_t dma_prot;
+
if (dev_is_dma_coherent(dev))
- return prot;
+ dma_prot = prot;
#ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE
- if (attrs & DMA_ATTR_WRITE_COMBINE)
- return pgprot_writecombine(prot);
+ else if (attrs & DMA_ATTR_WRITE_COMBINE)
+ dma_prot = pgprot_writecombine(prot);
#endif
- return pgprot_dmacoherent(prot);
+ else
+ dma_prot = pgprot_dmacoherent(prot);
+
+ if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED))
+ return pgprot_decrypted(dma_prot);
+ else
+ return pgprot_encrypted(dma_prot);
}
#endif /* CONFIG_MMU */
--
2.43.0
^ permalink raw reply related
* [PATCH v8 13/23] dma: swiotlb: track pool encryption state and honor DMA_ATTR_CC_SHARED
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jason Gunthorpe,
Jiri Pirko, Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
Teach swiotlb to distinguish between encrypted and decrypted bounce
buffer pools, and make allocation and mapping paths select a pool whose
state matches the requested DMA attributes.
Add a cc_shared flag to io_tlb_mem, initialize it for the default and
restricted pools, and propagate __DMA_ATTR_ALLOC_CC_SHARED into swiotlb
pool allocation. Reject swiotlb alloc/map requests when the selected pool
does not match the required encrypted/decrypted state.
Also return DMA addresses with the matching phys_to_dma_{encrypted,
unencrypted} helper so the DMA address encoding stays consistent with the
chosen pool.
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
include/linux/dma-direct.h | 10 +++
include/linux/swiotlb.h | 10 ++-
kernel/dma/direct.c | 13 ++-
kernel/dma/swiotlb.c | 177 ++++++++++++++++++++++++++++---------
4 files changed, 162 insertions(+), 48 deletions(-)
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index c249912456f9..94fad4e7c11e 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -77,6 +77,10 @@ static inline dma_addr_t dma_range_map_max(const struct bus_dma_region *map)
#ifndef phys_to_dma_unencrypted
#define phys_to_dma_unencrypted phys_to_dma
#endif
+
+#ifndef phys_to_dma_encrypted
+#define phys_to_dma_encrypted phys_to_dma
+#endif
#else
static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
{
@@ -90,6 +94,12 @@ static inline dma_addr_t phys_to_dma_unencrypted(struct device *dev,
{
return dma_addr_unencrypted(__phys_to_dma(dev, paddr));
}
+
+static inline dma_addr_t phys_to_dma_encrypted(struct device *dev,
+ phys_addr_t paddr)
+{
+ return dma_addr_encrypted(__phys_to_dma(dev, paddr));
+}
/*
* If memory encryption is supported, phys_to_dma will set the memory encryption
* bit in the DMA address, and dma_to_phys will clear it.
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index ea4c0a292dea..ee42f7588847 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -66,6 +66,7 @@ extern void __init swiotlb_update_mem_attributes(void);
* @node: Member of the IO TLB memory pool list.
* @rcu: RCU head for swiotlb_dyn_free().
* @transient: %true if transient memory pool.
+ * @cc_shared: %true if the pool memory is shared for confidential computing.
*/
struct io_tlb_pool {
phys_addr_t start;
@@ -81,6 +82,7 @@ struct io_tlb_pool {
struct list_head node;
struct rcu_head rcu;
bool transient;
+ bool cc_shared;
#endif
};
@@ -92,6 +94,7 @@ struct io_tlb_pool {
* @debugfs: The dentry to debugfs.
* @force_bounce: %true if swiotlb bouncing is forced
* @for_alloc: %true if the pool is used for memory allocation
+ * @cc_shared: %true if the pool memory is shared for confidential computing.
* @can_grow: %true if more pools can be allocated dynamically.
* @phys_limit: Maximum allowed physical address.
* @lock: Lock to synchronize changes to the list.
@@ -111,6 +114,7 @@ struct io_tlb_mem {
struct dentry *debugfs;
bool force_bounce;
bool for_alloc;
+ bool cc_shared;
#ifdef CONFIG_SWIOTLB_DYNAMIC
bool can_grow;
u64 phys_limit;
@@ -282,7 +286,8 @@ static inline void swiotlb_sync_single_for_cpu(struct device *dev,
extern void swiotlb_print_info(void);
#ifdef CONFIG_DMA_RESTRICTED_POOL
-struct page *swiotlb_alloc(struct device *dev, size_t size);
+struct page *swiotlb_alloc(struct device *dev, size_t size,
+ unsigned long attrs);
bool swiotlb_free(struct device *dev, struct page *page, size_t size);
void swiotlb_free_from_pool(struct device *dev,
phys_addr_t tlb_addr, struct io_tlb_pool *pool);
@@ -292,7 +297,8 @@ static inline bool is_swiotlb_for_alloc(struct device *dev)
return dev->dma_io_tlb_mem->for_alloc;
}
#else
-static inline struct page *swiotlb_alloc(struct device *dev, size_t size)
+static inline struct page *swiotlb_alloc(struct device *dev, size_t size,
+ unsigned long attrs)
{
return NULL;
}
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 6960ee1cfefe..9f013ff8e0f6 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -96,9 +96,10 @@ static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)
return ret;
}
-static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)
+static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size,
+ unsigned long attrs)
{
- struct page *page = swiotlb_alloc(dev, size);
+ struct page *page = swiotlb_alloc(dev, size, attrs);
if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
swiotlb_free(dev, page, size);
@@ -254,8 +255,12 @@ void *dma_direct_alloc(struct device *dev, size_t size,
}
if (is_swiotlb_for_alloc(dev)) {
- page = dma_direct_alloc_swiotlb(dev, size);
+ page = dma_direct_alloc_swiotlb(dev, size, attrs);
if (page) {
+ /*
+ * swiotlb allocations comes from pool already marked
+ * decrypted
+ */
mark_mem_decrypt = false;
goto setup_page;
}
@@ -401,7 +406,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
&ret, gfp, attrs);
if (is_swiotlb_for_alloc(dev)) {
- page = dma_direct_alloc_swiotlb(dev, size);
+ page = dma_direct_alloc_swiotlb(dev, size, attrs);
if (!page)
return NULL;
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 046ae92c4832..335e27bc1e1f 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -259,10 +259,21 @@ void __init swiotlb_update_mem_attributes(void)
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
unsigned long bytes;
+ /*
+ * if platform support memory encryption, swiotlb buffers are
+ * shared by default.
+ */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+ io_tlb_default_mem.cc_shared = true;
+ else
+ io_tlb_default_mem.cc_shared = false;
+
if (!mem->nslabs || mem->late_alloc)
return;
bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
- set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
+
+ if (io_tlb_default_mem.cc_shared)
+ set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
}
static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
@@ -505,8 +516,10 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
if (!mem->slots)
goto error_slots;
- set_memory_decrypted((unsigned long)vstart,
- (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+ if (io_tlb_default_mem.cc_shared)
+ set_memory_decrypted((unsigned long)vstart,
+ (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+
swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), vstart, nslabs, true,
nareas);
add_mem_pool(&io_tlb_default_mem, mem);
@@ -539,7 +552,9 @@ void __init swiotlb_exit(void)
tbl_size = PAGE_ALIGN(mem->end - mem->start);
slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
- set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+ if (io_tlb_default_mem.cc_shared)
+ set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+
if (mem->late_alloc) {
area_order = get_order(array_size(sizeof(*mem->areas),
mem->nareas));
@@ -563,6 +578,7 @@ void __init swiotlb_exit(void)
* @gfp: GFP flags for the allocation.
* @bytes: Size of the buffer.
* @phys_limit: Maximum allowed physical address of the buffer.
+ * @attrs: DMA attributes for the allocation.
*
* Allocate pages from the buddy allocator. If successful, make the allocated
* pages decrypted that they can be used for DMA.
@@ -570,9 +586,11 @@ void __init swiotlb_exit(void)
* Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
* if the allocated physical address was above @phys_limit.
*/
-static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
+static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes,
+ u64 phys_limit, unsigned long attrs)
{
unsigned int order = get_order(bytes);
+ bool cc_shared = attrs & __DMA_ATTR_ALLOC_CC_SHARED;
struct page *page;
phys_addr_t paddr;
void *vaddr;
@@ -588,13 +606,13 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
}
vaddr = phys_to_virt(paddr);
- if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
+ if (cc_shared && set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
goto error;
return page;
error:
/* Intentional leak if pages cannot be encrypted again. */
- if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
+ if (cc_shared && !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
__free_pages(page, order);
return NULL;
}
@@ -602,6 +620,7 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
/**
* swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
* @dev: Device for which a memory pool is allocated.
+ * @mem: SWIOTLB allocator for the pool.
* @bytes: Size of the buffer.
* @phys_limit: Maximum allowed physical address of the buffer.
* @gfp: GFP flags for the allocation.
@@ -609,25 +628,23 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
*
* Return: Allocated pages, or %NULL on allocation failure.
*/
-static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
+static struct page *swiotlb_alloc_tlb(struct device *dev,
+ struct io_tlb_mem *mem, size_t bytes,
u64 phys_limit, gfp_t gfp, void **vaddr)
{
struct page *page;
- unsigned long attrs = 0;
+ unsigned long attrs = mem->cc_shared ? __DMA_ATTR_ALLOC_CC_SHARED : 0;
*vaddr = NULL;
/*
* Allocate from the atomic pools if memory is encrypted and
* the allocation is atomic, because decrypting may block.
*/
- if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
+ if (!gfpflags_allow_blocking(gfp) && dev && mem->cc_shared) {
+
if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
return NULL;
- /* swiotlb considered decrypted by default */
- if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
- attrs = __DMA_ATTR_ALLOC_CC_SHARED;
-
return dma_alloc_from_pool(dev, bytes, vaddr, gfp,
attrs, dma_coherent_ok);
}
@@ -638,7 +655,7 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
else if (phys_limit <= DMA_BIT_MASK(32))
gfp |= __GFP_DMA32;
- while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit))) {
+ while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit, attrs))) {
if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
phys_limit < DMA_BIT_MASK(64) &&
!(gfp & (__GFP_DMA32 | __GFP_DMA)))
@@ -659,21 +676,25 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
* swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
* @vaddr: Virtual address of the buffer.
* @bytes: Size of the buffer.
+ * @cc_shared: true if @vaddr was allocated decrypted and must be
+ * re-encrypted before being freed
*/
-static void swiotlb_free_tlb(void *vaddr, size_t bytes)
+static void swiotlb_free_tlb(void *vaddr, size_t bytes, bool cc_shared)
{
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
dma_free_from_pool(NULL, vaddr, bytes))
return;
/* Intentional leak if pages cannot be encrypted again. */
- if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
+ if (!cc_shared ||
+ !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
__free_pages(virt_to_page(vaddr), get_order(bytes));
}
/**
* swiotlb_alloc_pool() - allocate a new IO TLB memory pool
* @dev: Device for which a memory pool is allocated.
+ * @mem: SWIOTLB allocator for the pool.
* @minslabs: Minimum number of slabs.
* @nslabs: Desired (maximum) number of slabs.
* @nareas: Number of areas.
@@ -687,8 +708,9 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes)
* Return: New memory pool, or %NULL on allocation failure.
*/
static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
- unsigned long minslabs, unsigned long nslabs,
- unsigned int nareas, u64 phys_limit, gfp_t gfp)
+ struct io_tlb_mem *mem, unsigned long minslabs,
+ unsigned long nslabs, unsigned int nareas, u64 phys_limit,
+ gfp_t gfp)
{
struct io_tlb_pool *pool;
unsigned int slot_order;
@@ -707,10 +729,11 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
if (!pool)
goto error;
pool->areas = (void *)pool + sizeof(*pool);
+ pool->cc_shared = mem->cc_shared;
tlb_size = nslabs << IO_TLB_SHIFT;
- while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp,
- &tlb_vaddr))) {
+ while (!(tlb = swiotlb_alloc_tlb(dev, mem, tlb_size,
+ phys_limit, gfp, &tlb_vaddr))) {
if (nslabs <= minslabs)
goto error_tlb;
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
@@ -729,7 +752,7 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
return pool;
error_slots:
- swiotlb_free_tlb(tlb_vaddr, tlb_size);
+ swiotlb_free_tlb(tlb_vaddr, tlb_size, mem->cc_shared);
error_tlb:
kfree(pool);
error:
@@ -746,7 +769,7 @@ static void swiotlb_dyn_alloc(struct work_struct *work)
container_of(work, struct io_tlb_mem, dyn_alloc);
struct io_tlb_pool *pool;
- pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
+ pool = swiotlb_alloc_pool(NULL, mem, IO_TLB_MIN_SLABS, default_nslabs,
default_nareas, mem->phys_limit, GFP_KERNEL);
if (!pool) {
pr_warn_ratelimited("Failed to allocate new pool");
@@ -767,7 +790,7 @@ static void swiotlb_dyn_free(struct rcu_head *rcu)
size_t tlb_size = pool->end - pool->start;
free_pages((unsigned long)pool->slots, get_order(slots_size));
- swiotlb_free_tlb(pool->vaddr, tlb_size);
+ swiotlb_free_tlb(pool->vaddr, tlb_size, pool->cc_shared);
kfree(pool);
}
@@ -1031,6 +1054,7 @@ static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
* @pool: Memory pool to be searched.
* @area_index: Index of the IO TLB memory area to be searched.
* @orig_addr: Original (non-bounced) IO buffer address.
+ * @tbl_dma_addr: DMA address of the bounce buffer.
* @alloc_size: Total requested size of the bounce buffer,
* including initial alignment padding.
* @alloc_align_mask: Required alignment of the allocated buffer.
@@ -1042,13 +1066,11 @@ static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
* Return: Index of the first allocated slot, or -1 on error.
*/
static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
- int area_index, phys_addr_t orig_addr, size_t alloc_size,
- unsigned int alloc_align_mask)
+ int area_index, phys_addr_t orig_addr, dma_addr_t tbl_dma_addr,
+ size_t alloc_size, unsigned int alloc_align_mask)
{
struct io_tlb_area *area = pool->areas + area_index;
unsigned long boundary_mask = dma_get_seg_boundary(dev);
- dma_addr_t tbl_dma_addr =
- phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
unsigned long max_slots = get_max_slots(boundary_mask);
unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
unsigned int nslots = nr_slots(alloc_size), stride;
@@ -1061,6 +1083,8 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
BUG_ON(!nslots);
BUG_ON(area_index >= pool->nareas);
+ tbl_dma_addr &= boundary_mask;
+
/*
* Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
* page-aligned in the absence of any other alignment requirements.
@@ -1172,6 +1196,7 @@ static int swiotlb_search_area(struct device *dev, int start_cpu,
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
+ dma_addr_t tbl_dma_addr;
int area_index;
int index = -1;
@@ -1180,9 +1205,15 @@ static int swiotlb_search_area(struct device *dev, int start_cpu,
if (cpu_offset >= pool->nareas)
continue;
area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
+
+ if (mem->cc_shared)
+ tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
+ else
+ tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
+
index = swiotlb_search_pool_area(dev, pool, area_index,
- orig_addr, alloc_size,
- alloc_align_mask);
+ orig_addr, tbl_dma_addr,
+ alloc_size, alloc_align_mask);
if (index >= 0) {
*retpool = pool;
break;
@@ -1212,6 +1243,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
+ dma_addr_t tbl_dma_addr;
unsigned long nslabs;
unsigned long flags;
u64 phys_limit;
@@ -1236,12 +1268,17 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
nslabs = nr_slots(alloc_size);
phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
- pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
+ pool = swiotlb_alloc_pool(dev, mem, nslabs, nslabs, 1, phys_limit,
GFP_NOWAIT);
if (!pool)
return -1;
- index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,
+ if (mem->cc_shared)
+ tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
+ else
+ tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
+
+ index = swiotlb_search_pool_area(dev, pool, 0, orig_addr, tbl_dma_addr,
alloc_size, alloc_align_mask);
if (index < 0) {
swiotlb_dyn_free(&pool->rcu);
@@ -1286,15 +1323,23 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
size_t alloc_size, unsigned int alloc_align_mask,
struct io_tlb_pool **retpool)
{
+ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
+ dma_addr_t tbl_dma_addr;
int start, i;
int index;
- *retpool = pool = &dev->dma_io_tlb_mem->defpool;
+ *retpool = pool = &mem->defpool;
+ if (mem->cc_shared)
+ tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
+ else
+ tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
+
i = start = raw_smp_processor_id() & (pool->nareas - 1);
do {
index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
- alloc_size, alloc_align_mask);
+ tbl_dma_addr, alloc_size,
+ alloc_align_mask);
if (index >= 0)
return index;
if (++i >= pool->nareas)
@@ -1377,9 +1422,19 @@ static unsigned long mem_used(struct io_tlb_mem *mem)
* any pre- or post-padding for alignment
* @alloc_align_mask: Required start and end alignment of the allocated buffer
* @dir: DMA direction
- * @attrs: Optional DMA attributes for the map operation
+ * @attrs: Optional DMA attributes for the map operation, updated
+ * to match the selected SWIOTLB pool
*
* Find and allocate a suitable sequence of IO TLB slots for the request.
+ * The device's SWIOTLB pool must match the device's current DMA encryption
+ * requirements. If the device requires decrypted DMA, bouncing is done through
+ * an unencrypted pool and the mapping is marked shared. If the device can DMA
+ * to encrypted memory, bouncing is done through an encrypted pool even when the
+ * original DMA address was unencrypted. Enabling encrypted DMA for a device is
+ * therefore expected to update its default io_tlb_mem to an encrypted pool, so
+ * later bounce mappings for both encrypted and decrypted original memory use
+ * that encrypted pool.
+ *
* The allocated space starts at an alignment specified by alloc_align_mask,
* and the size of the allocated space is rounded up so that the total amount
* of allocated space is a multiple of (alloc_align_mask + 1). If
@@ -1416,6 +1471,30 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
+ if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
+
+ /* swiotlb pool is incorrect for this device */
+ if (unlikely(mem->cc_shared != force_dma_unencrypted(dev)))
+ return (phys_addr_t)DMA_MAPPING_ERROR;
+
+ } else if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
+ /*
+ * On hosts with memory encryption, SWIOTLB-backed memory is
+ * unencrypted. DMA addresses returned for bounce buffers must
+ * therefore be marked unencrypted, even for devices that can
+ * address encrypted memory. This also preserves swiotlb=force
+ * behavior for those devices.
+ */
+ if (unlikely(!mem->cc_shared))
+ return (phys_addr_t)DMA_MAPPING_ERROR;
+ }
+
+ /* Force attrs to match the kind of memory in the pool */
+ if (mem->cc_shared)
+ *attrs |= DMA_ATTR_CC_SHARED;
+ else
+ *attrs &= ~DMA_ATTR_CC_SHARED;
+
/*
* The default swiotlb memory pool is allocated with PAGE_SIZE
* alignment. If a mapping is requested with larger alignment,
@@ -1613,8 +1692,11 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
- /* Ensure that the address returned is DMA'ble */
- dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
+ if (attrs & DMA_ATTR_CC_SHARED)
+ dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
+ else
+ dma_addr = phys_to_dma_encrypted(dev, swiotlb_addr);
+
if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC,
@@ -1778,7 +1860,7 @@ static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
#ifdef CONFIG_DMA_RESTRICTED_POOL
-struct page *swiotlb_alloc(struct device *dev, size_t size)
+struct page *swiotlb_alloc(struct device *dev, size_t size, unsigned long attrs)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
@@ -1789,6 +1871,9 @@ struct page *swiotlb_alloc(struct device *dev, size_t size)
if (!mem)
return NULL;
+ if (mem->cc_shared != !!(attrs & __DMA_ATTR_ALLOC_CC_SHARED))
+ return NULL;
+
align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
index = swiotlb_find_slots(dev, 0, size, align, &pool);
if (index == -1)
@@ -1864,12 +1949,20 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
kfree(mem);
return -ENOMEM;
}
+ /*
+ * if platform supports memory encryption,
+ * restricted mem pool is shared by default
+ */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
+ mem->cc_shared = true;
+ set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
+ rmem->size >> PAGE_SHIFT);
+ } else {
+ mem->cc_shared = false;
+ }
- set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
- rmem->size >> PAGE_SHIFT);
swiotlb_init_io_tlb_pool(pool, rmem->base, phys_to_virt(rmem->base),
- nslabs,
- false, nareas);
+ nslabs, false, nareas);
mem->force_bounce = true;
mem->for_alloc = true;
#ifdef CONFIG_SWIOTLB_DYNAMIC
--
2.43.0
^ permalink raw reply related
* [PATCH v8 12/23] dma: swiotlb: pass mapping attributes by reference
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jason Gunthorpe,
Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
Change swiotlb_tbl_map_single() to take the DMA mapping attributes by
reference and update the direct callers accordingly.
This is a preparatory change for a follow-up patch which updates the
attributes based on the selected swiotlb pool. Keeping the signature change
separate makes the follow-up patch easier to review.
No functional change in this patch.
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
drivers/iommu/dma-iommu.c | 2 +-
drivers/xen/swiotlb-xen.c | 2 +-
include/linux/swiotlb.h | 2 +-
kernel/dma/swiotlb.c | 6 +++---
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 83bf15ae9777..c42e40e60575 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1180,7 +1180,7 @@ static phys_addr_t iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys,
trace_swiotlb_bounced(dev, phys, size);
phys = swiotlb_tbl_map_single(dev, phys, size, iova_mask(iovad), dir,
- attrs);
+ &attrs);
/*
* Untrusted devices should not see padding areas with random leftover
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 2cbf2b588f5b..8c4abe65cd49 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -243,7 +243,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
*/
trace_swiotlb_bounced(dev, dev_addr, size);
- map = swiotlb_tbl_map_single(dev, phys, size, 0, dir, attrs);
+ map = swiotlb_tbl_map_single(dev, phys, size, 0, dir, &attrs);
if (map == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index c92ff6791595..ea4c0a292dea 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -238,7 +238,7 @@ static inline phys_addr_t default_swiotlb_limit(void)
phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
size_t mapping_size, unsigned int alloc_aligned_mask,
- enum dma_data_direction dir, unsigned long attrs);
+ enum dma_data_direction dir, unsigned long *attrs);
dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
size_t size, enum dma_data_direction dir, unsigned long attrs);
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 908de28aceb2..046ae92c4832 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1396,7 +1396,7 @@ static unsigned long mem_used(struct io_tlb_mem *mem)
*/
phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
size_t mapping_size, unsigned int alloc_align_mask,
- enum dma_data_direction dir, unsigned long attrs)
+ enum dma_data_direction dir, unsigned long *attrs)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
unsigned int offset;
@@ -1430,7 +1430,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
size = ALIGN(mapping_size + offset, alloc_align_mask + 1);
index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool);
if (index == -1) {
- if (!(attrs & DMA_ATTR_NO_WARN))
+ if (!(*attrs & DMA_ATTR_NO_WARN))
dev_warn_ratelimited(dev,
"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
size, mem->nslabs, mem_used(mem));
@@ -1609,7 +1609,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
- swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);
+ swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, &attrs);
if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
--
2.43.0
^ permalink raw reply related
* [PATCH v8 11/23] dma-pool: track decrypted atomic pools and select them via attrs
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko,
Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
Teach the atomic DMA pool code to distinguish between encrypted and
unencrypted pools, and make pool allocation select the matching pool based
on DMA attributes.
Introduce a dma_gen_pool wrapper that records whether a pool is
unencrypted, initialize that state when the atomic pools are created, and
use it when expanding and resizing the pools. Update dma_alloc_from_pool()
to take attrs and skip pools whose encrypted state does not match
__DMA_ATTR_ALLOC_CC_SHARED. Update dma_free_from_pool() accordingly.
Also pass __DMA_ATTR_ALLOC_CC_SHARED from the swiotlb atomic allocation
path so decrypted swiotlb allocations are taken from the correct atomic
pool.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
drivers/iommu/dma-iommu.c | 7 +-
include/linux/dma-map-ops.h | 2 +-
kernel/dma/direct.c | 14 ++-
kernel/dma/pool.c | 182 +++++++++++++++++++++++-------------
kernel/dma/swiotlb.c | 8 +-
5 files changed, 139 insertions(+), 74 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 6a45acbbfb0c..83bf15ae9777 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1660,9 +1660,14 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
{
bool coherent = dev_is_dma_coherent(dev);
int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
+ bool is_alloc_cc_shared = attrs & __DMA_ATTR_ALLOC_CC_SHARED;
struct page *page = NULL;
void *cpu_addr;
+ /* Not yet supported */
+ if (is_alloc_cc_shared)
+ return NULL;
+
gfp |= __GFP_ZERO;
if (gfpflags_allow_blocking(gfp) &&
@@ -1673,7 +1678,7 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
!gfpflags_allow_blocking(gfp) && !coherent) {
page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
- gfp, NULL);
+ gfp, attrs, NULL);
if (!page)
return NULL;
} else {
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index 137e015c1750..8fae2b7deb20 100644
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -212,7 +212,7 @@ void *dma_common_pages_remap(struct page **pages, size_t size, pgprot_t prot,
void dma_common_free_remap(void *cpu_addr, size_t size);
struct page *dma_alloc_from_pool(struct device *dev, size_t size,
- void **cpu_addr, gfp_t flags,
+ void **cpu_addr, gfp_t flags, unsigned long attrs,
bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
bool dma_free_from_pool(struct device *dev, void *start, size_t size);
bool dma_free_from_pool_page(struct device *dev, struct page *page, size_t size);
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 1b49ba5e0a60..6960ee1cfefe 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -154,7 +154,8 @@ static bool dma_direct_use_pool(struct device *dev, gfp_t gfp)
}
static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size,
- dma_addr_t *dma_handle, void **cpu_addr, gfp_t gfp)
+ dma_addr_t *dma_handle, void **cpu_addr, gfp_t gfp,
+ unsigned long attrs)
{
struct page *page;
u64 phys_limit;
@@ -163,7 +164,8 @@ static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size,
return NULL;
gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
- page = dma_alloc_from_pool(dev, size, cpu_addr, gfp, dma_coherent_ok);
+ page = dma_alloc_from_pool(dev, size, cpu_addr, gfp, attrs,
+ dma_coherent_ok);
if (!page)
return NULL;
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
@@ -240,11 +242,14 @@ void *dma_direct_alloc(struct device *dev, size_t size,
/*
* Remapping or decrypting memory may block, allocate the memory from
* the atomic pools instead if we aren't allowed block.
+ * FIXME: With CONFIG_DMA_DIRECT_REMAP, the pool is also mapped as
+ * DMA-coherent (non-cacheable). We may want to create a separate pool
+ * dedicated to CC_SHARED atomic allocations.
*/
if ((remap || (attrs & __DMA_ATTR_ALLOC_CC_SHARED)) &&
dma_direct_use_pool(dev, gfp)) {
page = dma_direct_alloc_from_pool(dev, size, dma_handle,
- &ret, gfp);
+ &ret, gfp, attrs);
return page ? ret : NULL;
}
@@ -392,7 +397,8 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp))
- return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp);
+ return dma_direct_alloc_from_pool(dev, size, dma_handle,
+ &ret, gfp, attrs);
if (is_swiotlb_for_alloc(dev)) {
page = dma_direct_alloc_swiotlb(dev, size);
diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c
index e981c1faaadf..00f422a1e896 100644
--- a/kernel/dma/pool.c
+++ b/kernel/dma/pool.c
@@ -12,12 +12,18 @@
#include <linux/set_memory.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
+#include <linux/cc_platform.h>
-static struct gen_pool *atomic_pool_dma __ro_after_init;
+struct dma_gen_pool {
+ bool cc_shared;
+ struct gen_pool *pool;
+};
+
+static struct dma_gen_pool atomic_pool_dma __ro_after_init;
static unsigned long pool_size_dma;
-static struct gen_pool *atomic_pool_dma32 __ro_after_init;
+static struct dma_gen_pool atomic_pool_dma32 __ro_after_init;
static unsigned long pool_size_dma32;
-static struct gen_pool *atomic_pool_kernel __ro_after_init;
+static struct dma_gen_pool atomic_pool_kernel __ro_after_init;
static unsigned long pool_size_kernel;
/* Size can be defined by the coherent_pool command line */
@@ -76,7 +82,7 @@ static bool cma_in_zone(gfp_t gfp)
return true;
}
-static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
+static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_size,
gfp_t gfp)
{
unsigned int order;
@@ -84,6 +90,7 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
bool leak_pages = false;
void *addr;
int ret = -ENOMEM;
+ pgprot_t prot __maybe_unused;
/* Cannot allocate larger than MAX_PAGE_ORDER */
order = min(get_order(pool_size), MAX_PAGE_ORDER);
@@ -102,8 +109,12 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
arch_dma_prep_coherent(page, pool_size);
#ifdef CONFIG_DMA_DIRECT_REMAP
- addr = dma_common_contiguous_remap(page, pool_size,
- pgprot_decrypted(pgprot_dmacoherent(PAGE_KERNEL)),
+ if (dma_pool->cc_shared)
+ prot = pgprot_decrypted(pgprot_dmacoherent(PAGE_KERNEL));
+ else
+ prot = pgprot_dmacoherent(PAGE_KERNEL);
+
+ addr = dma_common_contiguous_remap(page, pool_size, prot,
__builtin_return_address(0));
if (!addr)
goto free_page;
@@ -114,14 +125,17 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
* Memory in the atomic DMA pools must be unencrypted, the pools do not
* shrink so no re-encryption occurs in dma_direct_free().
*/
- ret = set_memory_decrypted((unsigned long)page_to_virt(page),
- 1 << order);
- if (ret) {
- leak_pages = true;
- goto remove_mapping;
+ if (dma_pool->cc_shared) {
+ ret = set_memory_decrypted((unsigned long)page_to_virt(page),
+ 1 << order);
+ if (ret) {
+ leak_pages = true;
+ goto remove_mapping;
+ }
}
- ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),
- pool_size, NUMA_NO_NODE);
+
+ ret = gen_pool_add_virt(dma_pool->pool, (unsigned long)addr,
+ page_to_phys(page), pool_size, NUMA_NO_NODE);
if (ret)
goto encrypt_mapping;
@@ -129,12 +143,10 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
return 0;
encrypt_mapping:
- ret = set_memory_encrypted((unsigned long)page_to_virt(page),
- 1 << order);
- if (WARN_ON_ONCE(ret)) {
- /* Decrypt succeeded but encrypt failed, purposely leak */
+ if (dma_pool->cc_shared &&
+ set_memory_encrypted((unsigned long)page_to_virt(page), 1 << order))
leak_pages = true;
- }
+
remove_mapping:
#ifdef CONFIG_DMA_DIRECT_REMAP
dma_common_free_remap(addr, pool_size);
@@ -146,46 +158,52 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
return ret;
}
-static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp)
+static void atomic_pool_resize(struct dma_gen_pool *dma_pool, gfp_t gfp)
{
- if (pool && gen_pool_avail(pool) < atomic_pool_size)
- atomic_pool_expand(pool, gen_pool_size(pool), gfp);
+ if (dma_pool->pool && gen_pool_avail(dma_pool->pool) < atomic_pool_size)
+ atomic_pool_expand(dma_pool, gen_pool_size(dma_pool->pool), gfp);
}
static void atomic_pool_work_fn(struct work_struct *work)
{
if (IS_ENABLED(CONFIG_ZONE_DMA))
- atomic_pool_resize(atomic_pool_dma,
+ atomic_pool_resize(&atomic_pool_dma,
GFP_KERNEL | GFP_DMA);
if (IS_ENABLED(CONFIG_ZONE_DMA32))
- atomic_pool_resize(atomic_pool_dma32,
+ atomic_pool_resize(&atomic_pool_dma32,
GFP_KERNEL | GFP_DMA32);
- atomic_pool_resize(atomic_pool_kernel, GFP_KERNEL);
+ atomic_pool_resize(&atomic_pool_kernel, GFP_KERNEL);
}
-static __init struct gen_pool *__dma_atomic_pool_init(size_t pool_size,
- gfp_t gfp)
+static __init struct dma_gen_pool *__dma_atomic_pool_init(struct dma_gen_pool *dma_pool,
+ size_t pool_size, gfp_t gfp)
{
- struct gen_pool *pool;
int ret;
- pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
- if (!pool)
+ dma_pool->pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
+ if (!dma_pool->pool)
return NULL;
- gen_pool_set_algo(pool, gen_pool_first_fit_order_align, NULL);
+ gen_pool_set_algo(dma_pool->pool, gen_pool_first_fit_order_align, NULL);
- ret = atomic_pool_expand(pool, pool_size, gfp);
+ /* if platform is using memory encryption atomic pools are by default shared. */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+ dma_pool->cc_shared = true;
+ else
+ dma_pool->cc_shared = false;
+
+ ret = atomic_pool_expand(dma_pool, pool_size, gfp);
if (ret) {
- gen_pool_destroy(pool);
+ gen_pool_destroy(dma_pool->pool);
+ dma_pool->pool = NULL;
pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
pool_size >> 10, &gfp);
return NULL;
}
pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",
- gen_pool_size(pool) >> 10, &gfp);
- return pool;
+ gen_pool_size(dma_pool->pool) >> 10, &gfp);
+ return dma_pool;
}
#ifdef CONFIG_ZONE_DMA32
@@ -211,21 +229,22 @@ static int __init dma_atomic_pool_init(void)
/* All memory might be in the DMA zone(s) to begin with */
if (has_managed_zone(ZONE_NORMAL)) {
- atomic_pool_kernel = __dma_atomic_pool_init(atomic_pool_size,
- GFP_KERNEL);
- if (!atomic_pool_kernel)
+ __dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size, GFP_KERNEL);
+ if (!atomic_pool_kernel.pool)
ret = -ENOMEM;
}
+
if (has_managed_dma()) {
- atomic_pool_dma = __dma_atomic_pool_init(atomic_pool_size,
- GFP_KERNEL | GFP_DMA);
- if (!atomic_pool_dma)
+ __dma_atomic_pool_init(&atomic_pool_dma, atomic_pool_size,
+ GFP_KERNEL | GFP_DMA);
+ if (!atomic_pool_dma.pool)
ret = -ENOMEM;
}
+
if (has_managed_dma32) {
- atomic_pool_dma32 = __dma_atomic_pool_init(atomic_pool_size,
- GFP_KERNEL | GFP_DMA32);
- if (!atomic_pool_dma32)
+ __dma_atomic_pool_init(&atomic_pool_dma32, atomic_pool_size,
+ GFP_KERNEL | GFP_DMA32);
+ if (!atomic_pool_dma32.pool)
ret = -ENOMEM;
}
@@ -234,19 +253,44 @@ static int __init dma_atomic_pool_init(void)
}
postcore_initcall(dma_atomic_pool_init);
-static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp)
+static inline struct dma_gen_pool *__dma_guess_pool(struct dma_gen_pool *first,
+ struct dma_gen_pool *second, struct dma_gen_pool *third)
{
- if (prev == NULL) {
+ if (first->pool)
+ return first;
+ if (second && second->pool)
+ return second;
+ if (third && third->pool)
+ return third;
+ return NULL;
+}
+
+static inline struct dma_gen_pool *dma_guess_pool(struct dma_gen_pool *prev,
+ gfp_t gfp)
+{
+ if (!prev) {
if (gfp & GFP_DMA)
- return atomic_pool_dma ?: atomic_pool_dma32 ?: atomic_pool_kernel;
+ return __dma_guess_pool(&atomic_pool_dma,
+ &atomic_pool_dma32,
+ &atomic_pool_kernel);
+
if (gfp & GFP_DMA32)
- return atomic_pool_dma32 ?: atomic_pool_dma ?: atomic_pool_kernel;
- return atomic_pool_kernel ?: atomic_pool_dma32 ?: atomic_pool_dma;
+ return __dma_guess_pool(&atomic_pool_dma32,
+ &atomic_pool_dma,
+ &atomic_pool_kernel);
+
+ return __dma_guess_pool(&atomic_pool_kernel,
+ &atomic_pool_dma32,
+ &atomic_pool_dma);
}
- if (prev == atomic_pool_kernel)
- return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma;
- if (prev == atomic_pool_dma32)
- return atomic_pool_dma;
+
+ if (prev == &atomic_pool_kernel)
+ return __dma_guess_pool(&atomic_pool_dma32,
+ &atomic_pool_dma, NULL);
+
+ if (prev == &atomic_pool_dma32)
+ return __dma_guess_pool(&atomic_pool_dma, NULL, NULL);
+
return NULL;
}
@@ -276,16 +320,20 @@ static struct page *__dma_alloc_from_pool(struct device *dev, size_t size,
}
struct page *dma_alloc_from_pool(struct device *dev, size_t size,
- void **cpu_addr, gfp_t gfp,
+ void **cpu_addr, gfp_t gfp, unsigned long attrs,
bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t))
{
- struct gen_pool *pool = NULL;
+ struct dma_gen_pool *dma_pool = NULL;
struct page *page;
bool pool_found = false;
- while ((pool = dma_guess_pool(pool, gfp))) {
+ while ((dma_pool = dma_guess_pool(dma_pool, gfp))) {
+
+ if (dma_pool->cc_shared != !!(attrs & __DMA_ATTR_ALLOC_CC_SHARED))
+ continue;
+
pool_found = true;
- page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,
+ page = __dma_alloc_from_pool(dev, size, dma_pool->pool, cpu_addr,
phys_addr_ok);
if (page)
return page;
@@ -300,12 +348,14 @@ struct page *dma_alloc_from_pool(struct device *dev, size_t size,
bool dma_free_from_pool(struct device *dev, void *start, size_t size)
{
- struct gen_pool *pool = NULL;
+ struct dma_gen_pool *dma_pool = NULL;
+
+ while ((dma_pool = dma_guess_pool(dma_pool, 0))) {
- while ((pool = dma_guess_pool(pool, 0))) {
- if (!gen_pool_has_addr(pool, (unsigned long)start, size))
+ if (!gen_pool_has_addr(dma_pool->pool, (unsigned long)start, size))
continue;
- gen_pool_free(pool, (unsigned long)start, size);
+
+ gen_pool_free(dma_pool->pool, (unsigned long)start, size);
return true;
}
@@ -337,7 +387,7 @@ static void dma_pool_find_phys(struct gen_pool *pool, struct gen_pool_chunk *chu
match->found = true;
}
-static bool dma_free_from_pool_phys(struct gen_pool *pool, phys_addr_t phys,
+static bool dma_free_from_pool_phys(struct dma_gen_pool *dma_pool, phys_addr_t phys,
size_t size)
{
struct dma_pool_phys_match match = {
@@ -345,11 +395,11 @@ static bool dma_free_from_pool_phys(struct gen_pool *pool, phys_addr_t phys,
.size = size,
};
- gen_pool_for_each_chunk(pool, dma_pool_find_phys, &match);
+ gen_pool_for_each_chunk(dma_pool->pool, dma_pool_find_phys, &match);
if (!match.found)
return false;
- gen_pool_free(pool, match.addr, size);
+ gen_pool_free(dma_pool->pool, match.addr, size);
return true;
}
@@ -359,14 +409,14 @@ static bool dma_free_from_pool_phys(struct gen_pool *pool, phys_addr_t phys,
*/
bool dma_free_from_pool_page(struct device *dev, struct page *page, size_t size)
{
- struct gen_pool *pool = NULL;
+ struct dma_gen_pool *dma_pool = NULL;
phys_addr_t phys = page_to_phys(page);
if (!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP))
return dma_free_from_pool(dev, page_address(page), size);
- while ((pool = dma_guess_pool(pool, 0))) {
- if (dma_free_from_pool_phys(pool, phys, size))
+ while ((dma_pool = dma_guess_pool(dma_pool, 0))) {
+ if (dma_free_from_pool_phys(dma_pool, phys, size))
return true;
}
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index d54154c165e5..908de28aceb2 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -613,9 +613,9 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
u64 phys_limit, gfp_t gfp, void **vaddr)
{
struct page *page;
+ unsigned long attrs = 0;
*vaddr = NULL;
-
/*
* Allocate from the atomic pools if memory is encrypted and
* the allocation is atomic, because decrypting may block.
@@ -624,8 +624,12 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
return NULL;
+ /* swiotlb considered decrypted by default */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+ attrs = __DMA_ATTR_ALLOC_CC_SHARED;
+
return dma_alloc_from_pool(dev, bytes, vaddr, gfp,
- dma_coherent_ok);
+ attrs, dma_coherent_ok);
}
gfp &= ~GFP_ZONEMASK;
--
2.43.0
^ permalink raw reply related
* [PATCH v8 10/23] dma-direct: use __DMA_ATTR_ALLOC_CC_SHARED in alloc/free paths
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jason Gunthorpe,
Jiri Pirko, Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
Propagate force_dma_unencrypted() into __DMA_ATTR_ALLOC_CC_SHARED in the
dma-direct allocation path and use the attribute to drive the related
decisions.
This updates dma_direct_alloc(), dma_direct_free(), and
dma_direct_alloc_pages() to fold the forced unencrypted case into attrs.
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 42 +++++++++++++++++++++++++++++++++---------
kernel/dma/mapping.c | 9 +++++++++
2 files changed, 42 insertions(+), 9 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 171c89fe99c9..1b49ba5e0a60 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -192,16 +192,22 @@ void *dma_direct_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
bool remap = false, set_uncached = false;
- bool mark_mem_decrypt = true;
+ bool mark_mem_decrypt = false;
struct page *page;
void *ret;
+ if (force_dma_unencrypted(dev))
+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
+
+ if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
+ mark_mem_decrypt = true;
+
size = PAGE_ALIGN(size);
if (attrs & DMA_ATTR_NO_WARN)
gfp |= __GFP_NOWARN;
- if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
- !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev))
+ if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==
+ DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev))
return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
if (!dev_is_dma_coherent(dev)) {
@@ -235,7 +241,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
* Remapping or decrypting memory may block, allocate the memory from
* the atomic pools instead if we aren't allowed block.
*/
- if ((remap || force_dma_unencrypted(dev)) &&
+ if ((remap || (attrs & __DMA_ATTR_ALLOC_CC_SHARED)) &&
dma_direct_use_pool(dev, gfp)) {
page = dma_direct_alloc_from_pool(dev, size, dma_handle,
&ret, gfp);
@@ -314,12 +320,22 @@ void dma_direct_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
{
phys_addr_t phys;
- bool mark_mem_encrypted = true;
+ bool mark_mem_encrypted = false;
struct io_tlb_pool *swiotlb_pool;
unsigned int page_order = get_order(size);
- if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
- !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) {
+ /*
+ * If the allocation used decrypted/shared backing pages, restore
+ * the encryption state on free.
+ */
+ if (force_dma_unencrypted(dev))
+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
+
+ if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
+ mark_mem_encrypted = true;
+
+ if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==
+ DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) {
/* cpu_addr is a struct page cookie, not a kernel address */
dma_free_contiguous(dev, cpu_addr, size);
return;
@@ -368,10 +384,14 @@ void dma_direct_free(struct device *dev, size_t size,
struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
{
+ unsigned long attrs = 0;
struct page *page;
void *ret;
- if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
+ if (force_dma_unencrypted(dev))
+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
+
+ if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp);
if (is_swiotlb_for_alloc(dev)) {
@@ -405,7 +425,11 @@ void dma_direct_free_pages(struct device *dev, size_t size,
phys_addr_t phys;
void *vaddr = page_address(page);
struct io_tlb_pool *swiotlb_pool;
- bool mark_mem_encrypted = true;
+ /*
+ * if the device had requested for an unencrypted buffer,
+ * convert it to encrypted on free
+ */
+ bool mark_mem_encrypted = force_dma_unencrypted(dev);
/* If page is not from an atomic pool, dma_free_from_pool_page() fails */
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 4fe04669e5e6..d2f70b6ccd0f 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -638,6 +638,15 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
if (WARN_ON_ONCE(flag & __GFP_COMP))
return NULL;
+ if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED)) {
+ trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag,
+ attrs);
+ return NULL;
+ }
+
+ if (force_dma_unencrypted(dev))
+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
+
if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {
trace_dma_alloc(dev, cpu_addr, *dma_handle, size,
DMA_BIDIRECTIONAL, flag, attrs);
--
2.43.0
^ permalink raw reply related
* [PATCH v8 09/23] dma-mapping: Add internal shared allocation attribute
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jason Gunthorpe
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
DMA_ATTR_CC_SHARED describes an existing DMA mapping whose backing memory
is already shared, or decrypted, for confidential computing. It is a
mapping attribute: callers use it to request a shared DMA address encoding
for memory that has already been prepared for shared DMA.
Allocation paths need a related but different state. Once the DMA core
decides that an allocation must use shared backing pages, the lower-level
allocation helpers need to select shared pools, decrypt newly allocated
pages, derive the DMA address with the shared-memory translation and
restore encryption on free. That state is internal to the DMA-mapping
implementation and should not be passed by drivers to dma_alloc_attrs().
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
Documentation/core-api/dma-attributes.rst | 29 +++++++++++++++++++++++
include/linux/dma-mapping.h | 8 +++++++
include/trace/events/dma.h | 3 ++-
3 files changed, 39 insertions(+), 1 deletion(-)
diff --git a/Documentation/core-api/dma-attributes.rst b/Documentation/core-api/dma-attributes.rst
index 123c8468d58f..eee743184acd 100644
--- a/Documentation/core-api/dma-attributes.rst
+++ b/Documentation/core-api/dma-attributes.rst
@@ -179,3 +179,32 @@ interface when building their uAPIs, when possible.
It must never be used in an in-kernel driver that only works with
kernel memory.
+
+DMA_ATTR_CC_SHARED
+------------------
+
+This attribute indicates that a DMA mapping is shared, or decrypted, for
+confidential computing guests. For normal system memory, the caller must
+already have marked the memory decrypted with set_memory_decrypted(). CPU
+PTEs for the mapping must use pgprot_decrypted(), and the same shared
+semantic may be passed to a vIOMMU when it sets up the IOPTE.
+
+This attribute describes an existing mapping. It does not allocate shared
+backing pages and must not be passed to dma_alloc_attrs(). For MMIO, use
+this together with DMA_ATTR_MMIO to indicate shared MMIO. Unless
+DMA_ATTR_MMIO is provided, the mapping requires a struct page.
+
+__DMA_ATTR_ALLOC_CC_SHARED
+--------------------------
+
+This is an internal DMA-mapping attribute for confidential computing guests.
+It is used by allocation paths after the DMA core has determined that the
+backing pages must be shared, or decrypted. For example, the direct DMA and
+SWIOTLB allocation paths use it to select shared DMA pools, decrypt newly
+allocated pages, derive DMA addresses using the shared-memory translation, and
+restore encryption on free.
+
+__DMA_ATTR_ALLOC_CC_SHARED differs from DMA_ATTR_CC_SHARED in that it is not
+a caller-visible DMA API attribute. DMA_ATTR_CC_SHARED describes an
+already-shared mapping and requires the caller to have prepared normal
+system memory before mapping it.
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index cc0823a99cfd..a3e880649fa4 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -103,6 +103,14 @@
*/
#define DMA_ATTR_CC_SHARED (1UL << 13)
+/*
+ * __DMA_ATTR_ALLOC_CC_SHARED: Internal DMA-mapping attribute used by
+ * allocation paths that create shared (decrypted) backing pages for
+ * confidential computing guests. Drivers must not pass this attribute to
+ * dma_alloc_attrs().
+ */
+#define __DMA_ATTR_ALLOC_CC_SHARED (1UL << 14)
+
/*
* A dma_addr_t can hold any valid DMA or bus address for the platform. It can
* be given to a device to use as a DMA source or target. It is specific to a
diff --git a/include/trace/events/dma.h b/include/trace/events/dma.h
index 31c9ddf72c9d..9df02c1511de 100644
--- a/include/trace/events/dma.h
+++ b/include/trace/events/dma.h
@@ -35,7 +35,8 @@ TRACE_DEFINE_ENUM(DMA_NONE);
{ DMA_ATTR_MMIO, "MMIO" }, \
{ DMA_ATTR_DEBUGGING_IGNORE_CACHELINES, "CACHELINES_OVERLAP" }, \
{ DMA_ATTR_REQUIRE_COHERENT, "REQUIRE_COHERENT" }, \
- { DMA_ATTR_CC_SHARED, "CC_SHARED" })
+ { DMA_ATTR_CC_SHARED, "CC_SHARED" }, \
+ { __DMA_ATTR_ALLOC_CC_SHARED, "ALLOC_CC_SHARED" })
DECLARE_EVENT_CLASS(dma_map,
TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr,
--
2.43.0
^ permalink raw reply related
* [PATCH v8 08/23] coco: arm64: s390: powerpc: Mark secure guests with CC_ATTR_GUEST_MEM_ENCRYPT
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
arm64 Realms, powerpc secure guests and s390 protected virtualization
guests currently report CC_ATTR_MEM_ENCRYPT, but not
CC_ATTR_GUEST_MEM_ENCRYPT.
These environments are all secure guest configurations. Their
force_dma_unencrypted() implementations also treat the same secure guest
state as requiring unencrypted DMA. Report CC_ATTR_GUEST_MEM_ENCRYPT for
the same condition so generic confidential-computing code can distinguish
guest memory encryption from host memory encryption instead of relying only
on CC_ATTR_MEM_ENCRYPT.
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
arch/arm64/mm/init.c | 1 +
arch/powerpc/platforms/pseries/cc_platform.c | 1 +
arch/s390/mm/init.c | 1 +
3 files changed, 3 insertions(+)
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index edcc305ec883..48d69e390158 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -419,6 +419,7 @@ bool cc_platform_has(enum cc_attr attr)
{
switch (attr) {
case CC_ATTR_MEM_ENCRYPT:
+ case CC_ATTR_GUEST_MEM_ENCRYPT:
return is_realm_world() || is_protected_kvm_guest();
default:
return false;
diff --git a/arch/powerpc/platforms/pseries/cc_platform.c b/arch/powerpc/platforms/pseries/cc_platform.c
index e8021af83a19..46e110a5a717 100644
--- a/arch/powerpc/platforms/pseries/cc_platform.c
+++ b/arch/powerpc/platforms/pseries/cc_platform.c
@@ -17,6 +17,7 @@ bool cc_platform_has(enum cc_attr attr)
{
switch (attr) {
case CC_ATTR_MEM_ENCRYPT:
+ case CC_ATTR_GUEST_MEM_ENCRYPT:
return is_secure_guest();
default:
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index c29326c44af0..6b1c5a4fa9ce 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -148,6 +148,7 @@ bool cc_platform_has(enum cc_attr attr)
{
switch (attr) {
case CC_ATTR_MEM_ENCRYPT:
+ case CC_ATTR_GUEST_MEM_ENCRYPT:
return is_prot_virt_guest();
default:
--
2.43.0
^ permalink raw reply related
* [PATCH v8 07/23] dma-direct: swiotlb: handle swiotlb alloc/free outside __dma_direct_alloc_pages
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jason Gunthorpe,
Jiri Pirko, Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
Move swiotlb allocation out of __dma_direct_alloc_pages() and handle it in
dma_direct_alloc() / dma_direct_alloc_pages().
This is needed for follow-up changes that simplify the handling of
memory encryption/decryption based on the DMA attribute flags.
swiotlb backing pages are already mapped decrypted by
swiotlb_update_mem_attributes() and rmem_swiotlb_device_init(), so
dma-direct should not call dma_set_decrypted() on allocation nor
dma_set_encrypted() on free for swiotlb-backed memory.
Update alloc/free paths to detect swiotlb-backed pages and skip
encrypt/decrypt transitions for those paths. Keep the existing highmem
rejection in dma_direct_alloc_pages() for swiotlb allocations.
Only for "restricted-dma-pool", we currently set `for_alloc = true`, while
rmem_swiotlb_device_init() decrypts the whole pool up front. This pool is
typically used together with "shared-dma-pool", where the shared region is
accessed after remap/ioremap and the returned address is suitable for
decrypted memory access. So existing code paths remain valid.
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
include/linux/swiotlb.h | 6 ++++
kernel/dma/direct.c | 71 ++++++++++++++++++++++++++++++-----------
kernel/dma/swiotlb.c | 6 ++++
3 files changed, 65 insertions(+), 18 deletions(-)
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 3dae0f592063..c92ff6791595 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -284,6 +284,8 @@ extern void swiotlb_print_info(void);
#ifdef CONFIG_DMA_RESTRICTED_POOL
struct page *swiotlb_alloc(struct device *dev, size_t size);
bool swiotlb_free(struct device *dev, struct page *page, size_t size);
+void swiotlb_free_from_pool(struct device *dev,
+ phys_addr_t tlb_addr, struct io_tlb_pool *pool);
static inline bool is_swiotlb_for_alloc(struct device *dev)
{
@@ -299,6 +301,10 @@ static inline bool swiotlb_free(struct device *dev, struct page *page,
{
return false;
}
+static inline void swiotlb_free_from_pool(struct device *dev,
+ phys_addr_t tlb_addr, struct io_tlb_pool *pool)
+{
+}
static inline bool is_swiotlb_for_alloc(struct device *dev)
{
return false;
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index a81597f8c3ed..171c89fe99c9 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -96,14 +96,6 @@ static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)
return ret;
}
-static void __dma_direct_free_pages(struct device *dev, struct page *page,
- size_t size)
-{
- if (swiotlb_free(dev, page, size))
- return;
- dma_free_contiguous(dev, page, size);
-}
-
static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)
{
struct page *page = swiotlb_alloc(dev, size);
@@ -125,9 +117,6 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
WARN_ON_ONCE(!PAGE_ALIGNED(size));
- if (is_swiotlb_for_alloc(dev))
- return dma_direct_alloc_swiotlb(dev, size);
-
gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
page = dma_alloc_contiguous(dev, size, gfp);
if (page) {
@@ -203,6 +192,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
bool remap = false, set_uncached = false;
+ bool mark_mem_decrypt = true;
struct page *page;
void *ret;
@@ -252,11 +242,21 @@ void *dma_direct_alloc(struct device *dev, size_t size,
return page ? ret : NULL;
}
+ if (is_swiotlb_for_alloc(dev)) {
+ page = dma_direct_alloc_swiotlb(dev, size);
+ if (page) {
+ mark_mem_decrypt = false;
+ goto setup_page;
+ }
+ return NULL;
+ }
+
/* we always manually zero the memory once we are done */
page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);
if (!page)
return NULL;
+setup_page:
/*
* dma_alloc_contiguous can return highmem pages depending on a
* combination the cma= arguments and per-arch setup. These need to be
@@ -283,7 +283,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
goto out_free_pages;
} else {
ret = page_address(page);
- if (dma_set_decrypted(dev, ret, size))
+ if (mark_mem_decrypt && dma_set_decrypted(dev, ret, size))
goto out_leak_pages;
}
@@ -300,10 +300,11 @@ void *dma_direct_alloc(struct device *dev, size_t size,
return ret;
out_encrypt_pages:
- if (dma_set_encrypted(dev, page_address(page), size))
+ if (mark_mem_decrypt && dma_set_encrypted(dev, page_address(page), size))
return NULL;
out_free_pages:
- __dma_direct_free_pages(dev, page, size);
+ if (!swiotlb_free(dev, page, size))
+ dma_free_contiguous(dev, page, size);
return NULL;
out_leak_pages:
return NULL;
@@ -312,6 +313,9 @@ void *dma_direct_alloc(struct device *dev, size_t size,
void dma_direct_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
{
+ phys_addr_t phys;
+ bool mark_mem_encrypted = true;
+ struct io_tlb_pool *swiotlb_pool;
unsigned int page_order = get_order(size);
if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
@@ -340,16 +344,25 @@ void dma_direct_free(struct device *dev, size_t size,
dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
return;
+ phys = dma_to_phys(dev, dma_addr);
+ swiotlb_pool = swiotlb_find_pool(dev, phys);
+ if (swiotlb_pool)
+ /* Swiotlb doesn't need a page attribute update on free */
+ mark_mem_encrypted = false;
+
if (is_vmalloc_addr(cpu_addr)) {
vunmap(cpu_addr);
} else {
if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
arch_dma_clear_uncached(cpu_addr, size);
- if (dma_set_encrypted(dev, cpu_addr, size))
+ if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size))
return;
}
- __dma_direct_free_pages(dev, dma_direct_to_page(dev, dma_addr), size);
+ if (swiotlb_pool)
+ swiotlb_free_from_pool(dev, phys, swiotlb_pool);
+ else
+ dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size);
}
struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
@@ -361,6 +374,15 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp);
+ if (is_swiotlb_for_alloc(dev)) {
+ page = dma_direct_alloc_swiotlb(dev, size);
+ if (!page)
+ return NULL;
+
+ ret = page_address(page);
+ goto setup_page;
+ }
+
page = __dma_direct_alloc_pages(dev, size, gfp, false);
if (!page)
return NULL;
@@ -368,6 +390,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
ret = page_address(page);
if (dma_set_decrypted(dev, ret, size))
goto out_leak_pages;
+setup_page:
memset(ret, 0, size);
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
return page;
@@ -379,16 +402,28 @@ void dma_direct_free_pages(struct device *dev, size_t size,
struct page *page, dma_addr_t dma_addr,
enum dma_data_direction dir)
{
+ phys_addr_t phys;
void *vaddr = page_address(page);
+ struct io_tlb_pool *swiotlb_pool;
+ bool mark_mem_encrypted = true;
/* If page is not from an atomic pool, dma_free_from_pool_page() fails */
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
dma_free_from_pool_page(dev, page, size))
return;
- if (dma_set_encrypted(dev, vaddr, size))
+ phys = page_to_phys(page);
+ swiotlb_pool = swiotlb_find_pool(dev, phys);
+ if (swiotlb_pool)
+ mark_mem_encrypted = false;
+
+ if (mark_mem_encrypted && dma_set_encrypted(dev, vaddr, size))
return;
- __dma_direct_free_pages(dev, page, size);
+
+ if (swiotlb_pool)
+ swiotlb_free_from_pool(dev, phys, swiotlb_pool);
+ else
+ dma_free_contiguous(dev, page, size);
}
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 6e8db52866bf..d54154c165e5 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1815,6 +1815,12 @@ bool swiotlb_free(struct device *dev, struct page *page, size_t size)
return true;
}
+void swiotlb_free_from_pool(struct device *dev,
+ phys_addr_t tlb_addr, struct io_tlb_pool *pool)
+{
+ swiotlb_release_slots(dev, tlb_addr, pool);
+}
+
static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
struct device *dev)
{
--
2.43.0
^ permalink raw reply related
* [PATCH v8 06/23] s390: Expose protected virtualization through cc_platform_has()
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Halil Pasic,
Matthew Rosato, Jaehoon Kim
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
Protected virtualization guests use memory encryption, so advertise that to
the rest of the kernel through cc_platform_has(CC_ATTR_MEM_ENCRYPT).
s390 already forces DMA mappings to be unencrypted for protected
virtualization guests through force_dma_unencrypted(). Add
ARCH_HAS_CC_PLATFORM and provide the matching cc_platform_has()
implementation
Cc: Halil Pasic <pasic@linux.ibm.com>
Cc: Matthew Rosato <mjrosato@linux.ibm.com>
Cc: Jaehoon Kim <jhkim@linux.ibm.com>
Tested-by: Jaehoon Kim <jhkim@linux.ibm.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
arch/s390/Kconfig | 1 +
arch/s390/mm/init.c | 14 ++++++++++++++
2 files changed, 15 insertions(+)
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 84404e6778d5..ab1bf55d197a 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -84,6 +84,7 @@ config S390
select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
select ARCH_HAS_CC_CAN_LINK
+ select ARCH_HAS_CC_PLATFORM
select ARCH_HAS_CPU_FINALIZE_INIT
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index f07168a0d3dd..c29326c44af0 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -50,6 +50,7 @@
#include <linux/virtio_anchor.h>
#include <linux/virtio_config.h>
#include <linux/execmem.h>
+#include <linux/cc_platform.h>
pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir");
pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir");
@@ -142,6 +143,19 @@ bool force_dma_unencrypted(struct device *dev)
return is_prot_virt_guest();
}
+
+bool cc_platform_has(enum cc_attr attr)
+{
+ switch (attr) {
+ case CC_ATTR_MEM_ENCRYPT:
+ return is_prot_virt_guest();
+
+ default:
+ return false;
+ }
+}
+EXPORT_SYMBOL_GPL(cc_platform_has);
+
/* protected virtualization */
static void __init pv_init(void)
{
--
2.43.0
^ permalink raw reply related
* [PATCH v8 05/23] swiotlb: Preserve allocation virtual address for dynamic pools
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jason Gunthorpe,
Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
swiotlb_alloc_tlb() can allocate from the DMA atomic pool when a decrypted
pool is needed from atomic context. With CONFIG_DMA_DIRECT_REMAP, the
atomic pool is backed by remapped virtual addresses, which are not the same
as the direct-map addresses returned by phys_to_virt().
swiotlb_init_io_tlb_pool() currently reconstructs the pool virtual address
from the physical start address. For atomic-pool backed allocations this
stores the wrong address in pool->vaddr. Later, swiotlb_free_tlb() passes
that address to dma_free_from_pool(), which will fail to recognize the
chunk
Pass the virtual address returned by the allocation path into
swiotlb_init_io_tlb_pool(), and store that address in pool->vaddr. This
keeps the pool free path using the same virtual address as the allocator.
Fixes: 79636caad361 ("swiotlb: if swiotlb is full, fall back to a transient memory pool")
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/swiotlb.c | 31 +++++++++++++++++++------------
1 file changed, 19 insertions(+), 12 deletions(-)
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 1abd3e6146f4..6e8db52866bf 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -266,9 +266,9 @@ void __init swiotlb_update_mem_attributes(void)
}
static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
- unsigned long nslabs, bool late_alloc, unsigned int nareas)
+ void *vaddr, unsigned long nslabs, bool late_alloc,
+ unsigned int nareas)
{
- void *vaddr = phys_to_virt(start);
unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
mem->nslabs = nslabs;
@@ -409,7 +409,7 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
return;
}
- swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas);
+ swiotlb_init_io_tlb_pool(mem, __pa(tlb), tlb, nslabs, false, nareas);
add_mem_pool(&io_tlb_default_mem, mem);
if (flags & SWIOTLB_VERBOSE)
@@ -507,7 +507,7 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
set_memory_decrypted((unsigned long)vstart,
(nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
- swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true,
+ swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), vstart, nslabs, true,
nareas);
add_mem_pool(&io_tlb_default_mem, mem);
@@ -605,25 +605,26 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
* @bytes: Size of the buffer.
* @phys_limit: Maximum allowed physical address of the buffer.
* @gfp: GFP flags for the allocation.
+ * @vaddr: Receives the virtual address for the allocated buffer.
*
* Return: Allocated pages, or %NULL on allocation failure.
*/
static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
- u64 phys_limit, gfp_t gfp)
+ u64 phys_limit, gfp_t gfp, void **vaddr)
{
struct page *page;
+ *vaddr = NULL;
+
/*
* Allocate from the atomic pools if memory is encrypted and
* the allocation is atomic, because decrypting may block.
*/
if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
- void *vaddr;
-
if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
return NULL;
- return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
+ return dma_alloc_from_pool(dev, bytes, vaddr, gfp,
dma_coherent_ok);
}
@@ -645,6 +646,8 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
return NULL;
}
+ if (page)
+ *vaddr = phys_to_virt(page_to_phys(page));
return page;
}
@@ -685,6 +688,7 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
{
struct io_tlb_pool *pool;
unsigned int slot_order;
+ void *tlb_vaddr;
struct page *tlb;
size_t pool_size;
size_t tlb_size;
@@ -701,7 +705,8 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
pool->areas = (void *)pool + sizeof(*pool);
tlb_size = nslabs << IO_TLB_SHIFT;
- while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {
+ while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp,
+ &tlb_vaddr))) {
if (nslabs <= minslabs)
goto error_tlb;
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
@@ -715,11 +720,12 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
if (!pool->slots)
goto error_slots;
- swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas);
+ swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), tlb_vaddr, nslabs,
+ true, nareas);
return pool;
error_slots:
- swiotlb_free_tlb(page_address(tlb), tlb_size);
+ swiotlb_free_tlb(tlb_vaddr, tlb_size);
error_tlb:
kfree(pool);
error:
@@ -1851,7 +1857,8 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
rmem->size >> PAGE_SHIFT);
- swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs,
+ swiotlb_init_io_tlb_pool(pool, rmem->base, phys_to_virt(rmem->base),
+ nslabs,
false, nareas);
mem->force_bounce = true;
mem->for_alloc = true;
--
2.43.0
^ permalink raw reply related
* [PATCH v8 04/23] dma: free atomic pool pages by physical address
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
dma_direct_alloc_pages() may satisfy atomic allocations from the coherent
atomic pools. The pool allocation is keyed by the virtual address stored in
the gen_pool, but the pages API returns only the backing struct page.
On architectures with CONFIG_DMA_DIRECT_REMAP, atomic pool chunks are added
to the gen_pool using their remapped virtual address.
dma_direct_free_pages() reconstructs a linear-map address with
page_address(page) and passes that to dma_free_from_pool(). That address
does not match the gen_pool virtual range, so the pool lookup can fail and
the code can fall through to freeing a pool-owned page through the normal
page allocator path.
Add a page-based pool free helper that looks up the owning pool chunk by
physical address, translates it back to the gen_pool virtual address, and
frees that address to the pool. Use it from dma_direct_free_pages() while
keeping the existing virtual-address helper for coherent allocation frees.
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
include/linux/dma-map-ops.h | 1 +
kernel/dma/direct.c | 4 +--
kernel/dma/pool.c | 61 +++++++++++++++++++++++++++++++++++++
3 files changed, 64 insertions(+), 2 deletions(-)
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index bcb5b5428aea..137e015c1750 100644
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -215,6 +215,7 @@ struct page *dma_alloc_from_pool(struct device *dev, size_t size,
void **cpu_addr, gfp_t flags,
bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
bool dma_free_from_pool(struct device *dev, void *start, size_t size);
+bool dma_free_from_pool_page(struct device *dev, struct page *page, size_t size);
int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start,
dma_addr_t dma_start, u64 size);
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 363d984d90e7..a81597f8c3ed 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -381,9 +381,9 @@ void dma_direct_free_pages(struct device *dev, size_t size,
{
void *vaddr = page_address(page);
- /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */
+ /* If page is not from an atomic pool, dma_free_from_pool_page() fails */
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
- dma_free_from_pool(dev, vaddr, size))
+ dma_free_from_pool_page(dev, page, size))
return;
if (dma_set_encrypted(dev, vaddr, size))
diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c
index b0303efbc153..e981c1faaadf 100644
--- a/kernel/dma/pool.c
+++ b/kernel/dma/pool.c
@@ -311,3 +311,64 @@ bool dma_free_from_pool(struct device *dev, void *start, size_t size)
return false;
}
+
+struct dma_pool_phys_match {
+ phys_addr_t phys;
+ size_t size;
+ unsigned long addr;
+ bool found;
+};
+
+static void dma_pool_find_phys(struct gen_pool *pool, struct gen_pool_chunk *chunk,
+ void *data)
+{
+ struct dma_pool_phys_match *match = data;
+ phys_addr_t end = match->phys + match->size - 1;
+ phys_addr_t chunk_end;
+
+ if (match->found)
+ return;
+
+ chunk_end = chunk->phys_addr + (chunk->end_addr - chunk->start_addr);
+ if (match->phys < chunk->phys_addr || end > chunk_end)
+ return;
+
+ match->addr = chunk->start_addr + (match->phys - chunk->phys_addr);
+ match->found = true;
+}
+
+static bool dma_free_from_pool_phys(struct gen_pool *pool, phys_addr_t phys,
+ size_t size)
+{
+ struct dma_pool_phys_match match = {
+ .phys = phys,
+ .size = size,
+ };
+
+ gen_pool_for_each_chunk(pool, dma_pool_find_phys, &match);
+ if (!match.found)
+ return false;
+
+ gen_pool_free(pool, match.addr, size);
+ return true;
+}
+
+/*
+ * FIXME: We could avoid this by storing the remapped virtual address in
+ * struct page and using that for lookup.
+ */
+bool dma_free_from_pool_page(struct device *dev, struct page *page, size_t size)
+{
+ struct gen_pool *pool = NULL;
+ phys_addr_t phys = page_to_phys(page);
+
+ if (!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP))
+ return dma_free_from_pool(dev, page_address(page), size);
+
+ while ((pool = dma_guess_pool(pool, 0))) {
+ if (dma_free_from_pool_phys(pool, phys, size))
+ return true;
+ }
+
+ return false;
+}
--
2.43.0
^ permalink raw reply related
* [PATCH v8 03/23] iommu/dma: Check atomic pool allocation result directly
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jason Gunthorpe,
Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
The non-blocking, non-coherent allocation path uses dma_alloc_from_pool(),
which returns the allocated page and fills cpu_addr only on success.
Do not rely on cpu_addr to detect allocation failure in this path. Check
the returned page directly before using it for the IOMMU mapping.
Fixes: 9420139f516d ("dma-pool: fix coherent pool allocations for IOMMU mappings")
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
drivers/iommu/dma-iommu.c | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 5598ed4bff72..6a45acbbfb0c 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1671,13 +1671,16 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
}
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
- !gfpflags_allow_blocking(gfp) && !coherent)
+ !gfpflags_allow_blocking(gfp) && !coherent) {
page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
- gfp, NULL);
- else
+ gfp, NULL);
+ if (!page)
+ return NULL;
+ } else {
cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
- if (!cpu_addr)
- return NULL;
+ if (!cpu_addr)
+ return NULL;
+ }
*handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot,
dev->coherent_dma_mask);
--
2.43.0
^ permalink raw reply related
* Re: [PATCH v5 1/4] arm64: vdso: Prepare for robust futex unlock support
From: Mark Rutland @ 2026-07-17 18:05 UTC (permalink / raw)
To: André Almeida
Cc: Catalin Marinas, Will Deacon, Thomas Gleixner, Mathieu Desnoyers,
Sebastian Andrzej Siewior, Carlos O'Donell, Peter Zijlstra,
Florian Weimer, Rich Felker, Torvald Riegel, Darren Hart,
Ingo Molnar, Davidlohr Bueso, Arnd Bergmann, Uros Bizjak,
Thomas Weißschuh, Liam R. Howlett, linux-arm-kernel,
linux-kernel, linux-arch, kernel-dev
In-Reply-To: <20260717-tonyk-robust_arm-v5-1-ffd1ad318d17@igalia.com>
Hi Andre,
On Fri, Jul 17, 2026 at 11:41:40AM -0300, André Almeida wrote:
> There will be a VDSO function to unlock non-contended robust futexes in
> user space. The unlock sequence is racy vs. clearing the list_pending_op
> pointer in the task's robust list head. To plug this race the kernel needs
> to know the critical section window so it can clear the pointer when the
> task is interrupted within that race window. The window is determined by
> labels in the inline assembly.
>
> Signed-off-by: André Almeida <andrealmeid@igalia.com>
I think this patch is confusing, because half of the additions are
tightly coupled with the AArch64-specific changes in the second patch.
I think it would be better to have preparatory patches which:
(1) Split vdso_mremap() into vdso_mremap(), and aarch32_mremap(). No
functional change, no other additions. Possibly have a common
__vdso_mremap() helper.
(2) If necessary, add only the common helper function(s) here, no native
symbols, etc. anything specific to the native vDSO should be in the
next patch, which adds the assembly etc.
See below for more on that.
> ---
> v4:
> - Guard symbols from vdso.lds.S with ifdef
> - drop update_ips() from sigpage remap function
>
> v3:
> - Fix adding vdso base addr twice
> - Call vdso_futex_robust_unlock_update_ips() on remap as well
> v2:
> - Fixed linker not finding VDSO symbols
> ---
> arch/arm64/kernel/vdso.c | 35 ++++++++++++++++++++++++++++++++++-
> arch/arm64/kernel/vdso/vdso.lds.S | 9 +++++++++
> 2 files changed, 43 insertions(+), 1 deletion(-)
>
> diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
> index 592dd8668de4..ff43b74e514e 100644
> --- a/arch/arm64/kernel/vdso.c
> +++ b/arch/arm64/kernel/vdso.c
> @@ -11,6 +11,7 @@
> #include <linux/clocksource.h>
> #include <linux/elf.h>
> #include <linux/err.h>
> +#include <linux/futex.h>
> #include <linux/errno.h>
> #include <linux/gfp.h>
> #include <linux/kernel.h>
> @@ -57,11 +58,41 @@ static struct vdso_abi_info vdso_info[] __ro_after_init = {
> #endif /* CONFIG_COMPAT_VDSO */
> };
>
> +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
> +static void vdso_futex_robust_unlock_update_ips(enum vdso_abi abi, struct mm_struct *mm)
> +{
> + unsigned long vdso = (unsigned long) mm->context.vdso;
> + struct futex_mm_data *fd = &mm->futex;
> + uintptr_t success, end;
> +
> + if (abi == VDSO_ABI_AA64) {
> + success = (uintptr_t) VDSO_SYMBOL(vdso, futex_list64_try_unlock_cs_start);
> + end = (uintptr_t) VDSO_SYMBOL(vdso, futex_list64_try_unlock_cs_end);
Is this supposed to be the success label, or the start label?
I see x86 uses 'start', so I assume s/success/start/ here.
> +
> + futex_set_vdso_cs_range(fd, 0, success, end, false);
> + }
> +}
I think it would be better to have a structure of functions along the
lines of a common helper:
| static inline void __vdso_futex_update_ips(struct mm_struct *mm, bool is_32bit,
| void *startp,
| void *endp)
| {
| unsigned long start = (unsigned long)startp;
| unsigned long end = (unsigned long)endp;
|
| struct futex_mm_data *fd = &mm->futex;
| futex_set_vdso_cs_range(fd, 0, start, end, is_32bit);
| }
... and then subsequent patches can add the native/compat cases as
callers of that common helper:
| static inline void vdso_futex_update_ips(struct mm_struct *mm)
| {
| unsigned long vdso = (unsigned long)mm->context.vdso;
| __vdso_futex_update_ips(mm, false,
| VDSO_SYMBOL(vdso, futex_list64_try_unlock_cs_start),
| VDSO_SYMBOL(vdso, futex_list64_try_unlock_cs_end));
|
| }
|
| static inline void aarch32_vdso_futex_update_ips(struct mm_struct *mm)
| {
| unsigned long vdso = (unsigned long)mm->context.vdso;
| __vdso_futex_update_ips(mm, true,
| VDSO_SYMBOL(vdso, futex_list32_try_unlock_cs_start),
| VDSO_SYMBOL(vdso, futex_list32_try_unlock_cs_end));
|
| }
> +#else
> +static inline void vdso_futex_robust_unlock_update_ips(enum vdso_abi abi, struct mm_struct *mm) { }
> +#endif /* CONFIG_FUTEX_ROBUST_UNLOCK */
> +
> static int vdso_mremap(const struct vm_special_mapping *sm,
> struct vm_area_struct *new_vma)
> {
> current->mm->context.vdso = (void *)new_vma->vm_start;
>
> + vdso_futex_robust_unlock_update_ips(VDSO_ABI_AA64, current->mm);
> +
> + return 0;
> +}
> +
> +static int vdso32_mremap(const struct vm_special_mapping *sm,
> + struct vm_area_struct *new_vma)
> +{
> + current->mm->context.vdso = (void *)new_vma->vm_start;
> +
> + vdso_futex_robust_unlock_update_ips(VDSO_ABI_AA32, current->mm);
> +
> return 0;
> }
As above, I think we should introduce this as a separate patch, and for
consistency with other parts of this file, the prefix should be
'aarch32_' rather than 'vdso32_'
>
> @@ -134,6 +165,8 @@ static int __setup_additional_pages(enum vdso_abi abi,
> if (IS_ERR(ret))
> goto up_fail;
>
> + vdso_futex_robust_unlock_update_ips(abi, mm);
> +
As above, I think this shouldn't be called here.
The call for the native vdso should be added to
arch_setup_additional_pages() along with the other additions for the
native vdso. The call for the compat vdso should be added to
aarch32_setup_additional_pages().
[...]
> diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S
> index 52314be29191..225f59bb81d1 100644
> --- a/arch/arm64/kernel/vdso/vdso.lds.S
> +++ b/arch/arm64/kernel/vdso/vdso.lds.S
> @@ -104,6 +104,9 @@ VERSION
> __kernel_clock_gettime;
> __kernel_clock_getres;
> __kernel_getrandom;
> +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
> + __vdso_futex_robust_list64_try_unlock;
> +#endif
> local: *;
> };
> }
> @@ -112,3 +115,9 @@ VERSION
> * Make the sigreturn code visible to the kernel.
> */
> VDSO_sigtramp = __kernel_rt_sigreturn;
> +
> +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
> +VDSO_futex_list64_try_unlock_cs_start = __futex_list64_try_unlock_cs_start;
> +VDSO_futex_list64_try_unlock_cs_success = __futex_list64_try_unlock_cs_success;
> +VDSO_futex_list64_try_unlock_cs_end = __futex_list64_try_unlock_cs_end;
> +#endif
These additions should all be in the next patch which actually adds the
native implementation.
Mark.
^ permalink raw reply
* [PATCH v8 02/23] dma-pool: fix page leak in atomic_pool_expand() cleanup
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jason Gunthorpe,
Michael Kelley
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
atomic_pool_expand() frees the allocated pages from the remove_mapping
error path only when CONFIG_DMA_DIRECT_REMAP is enabled.
When CONFIG_DMA_DIRECT_REMAP is disabled, failures after page allocation,
such as gen_pool_add_virt(), jump to remove_mapping and return without
freeing the pages.
Move __free_pages(page, order) out of the CONFIG_DMA_DIRECT_REMAP block so
that cleanup paths always release the allocation.
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/pool.c | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c
index 2b2fbb709242..b0303efbc153 100644
--- a/kernel/dma/pool.c
+++ b/kernel/dma/pool.c
@@ -81,6 +81,7 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
{
unsigned int order;
struct page *page = NULL;
+ bool leak_pages = false;
void *addr;
int ret = -ENOMEM;
@@ -115,8 +116,10 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
*/
ret = set_memory_decrypted((unsigned long)page_to_virt(page),
1 << order);
- if (ret)
+ if (ret) {
+ leak_pages = true;
goto remove_mapping;
+ }
ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),
pool_size, NUMA_NO_NODE);
if (ret)
@@ -130,14 +133,15 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
1 << order);
if (WARN_ON_ONCE(ret)) {
/* Decrypt succeeded but encrypt failed, purposely leak */
- goto out;
+ leak_pages = true;
}
remove_mapping:
#ifdef CONFIG_DMA_DIRECT_REMAP
dma_common_free_remap(addr, pool_size);
free_page:
- __free_pages(page, order);
#endif
+ if (!leak_pages)
+ __free_pages(page, order);
out:
return ret;
}
--
2.43.0
^ permalink raw reply related
* [PATCH v8 01/23] dma-direct: return struct page from dma_direct_alloc_from_pool()
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, stable, Michael Kelley,
Jason Gunthorpe
In-Reply-To: <20260717180442.110954-1-aneesh.kumar@kernel.org>
Commit 5b138c534fda ("dma-direct: factor out a dma_direct_alloc_from_pool
helper") changed dma_direct_alloc_from_pool() to return the CPU address
from dma_alloc_from_pool(). That fits dma_direct_alloc(), but
dma_direct_alloc_pages() also uses the helper and expects a struct page *.
Fix this by making dma_direct_alloc_from_pool() return the struct page *
again, and pass the CPU address back through an out-parameter for the
dma_direct_alloc() caller.
Fixes: 5b138c534fda ("dma-direct: factor out a dma_direct_alloc_from_pool helper")
Cc: stable@vger.kernel.org
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index d8219efe3273..363d984d90e7 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -164,22 +164,21 @@ static bool dma_direct_use_pool(struct device *dev, gfp_t gfp)
return !gfpflags_allow_blocking(gfp) && !is_swiotlb_for_alloc(dev);
}
-static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
- dma_addr_t *dma_handle, gfp_t gfp)
+static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size,
+ dma_addr_t *dma_handle, void **cpu_addr, gfp_t gfp)
{
struct page *page;
u64 phys_limit;
- void *ret;
if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)))
return NULL;
gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
- page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);
+ page = dma_alloc_from_pool(dev, size, cpu_addr, gfp, dma_coherent_ok);
if (!page)
return NULL;
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
- return ret;
+ return page;
}
static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,
@@ -247,8 +246,11 @@ void *dma_direct_alloc(struct device *dev, size_t size,
* the atomic pools instead if we aren't allowed block.
*/
if ((remap || force_dma_unencrypted(dev)) &&
- dma_direct_use_pool(dev, gfp))
- return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+ dma_direct_use_pool(dev, gfp)) {
+ page = dma_direct_alloc_from_pool(dev, size, dma_handle,
+ &ret, gfp);
+ return page ? ret : NULL;
+ }
/* we always manually zero the memory once we are done */
page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);
@@ -357,7 +359,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
void *ret;
if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
- return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+ return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp);
page = __dma_direct_alloc_pages(dev, size, gfp, false);
if (!page)
--
2.43.0
^ permalink raw reply related
* [PATCH v8 00/23] dma-mapping: Track shared DMA state through direct, pool and swiotlb paths
From: Aneesh Kumar K.V (Arm) @ 2026-07-17 18:04 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86
This series tracks confidential-computing shared DMA state through the
dma-direct, dma-pool, and swiotlb paths so that encrypted and decrypted
DMA buffers are handled consistently.
Today, the direct DMA path mostly relies on force_dma_unencrypted() for
shared/decrypted buffer handling. This series consolidates the
force_dma_unencrypted() checks in the top-level functions and ensures
that the remaining DMA interfaces use DMA attributes to make the correct
decisions.
The series separates mapping and allocation state:
- DMA_ATTR_CC_SHARED describes the DMA address attribute requested for a
mapping. It tells the DMA mapping path that the DMA address must target
shared/decrypted memory.
- __DMA_ATTR_ALLOC_CC_SHARED is an internal DMA-mapping attribute used only
by allocation paths after the DMA core decides that the backing pages
must be allocated as shared/decrypted memory.
The series:
- moves swiotlb-backed allocations out of __dma_direct_alloc_pages(),
- uses __DMA_ATTR_ALLOC_CC_SHARED through the dma-direct alloc/free paths
- teaches the atomic DMA pools to track encrypted versus decrypted
state
- tracks swiotlb pool encryption state and enforces strict pool
selection
- centralizes encrypted/decrypted pgprot handling in dma_pgprot() using
DMA attributes
- passes DMA attributes down to dma_capable() so capability checks can
validate whether the selected DMA address encoding matches
DMA_ATTR_CC_SHARED
- makes dma_direct_map_phys() choose the DMA address encoding from
DMA_ATTR_CC_SHARED and fall back to swiotlb when a shared DMA request
cannot use the direct mapping, which lets arm64 and x86 CCA guests stop
relying on SWIOTLB_FORCE for DMA mappings
- use the selected swiotlb pool state to derive the returned DMA
address
- reports CC_ATTR_GUEST_MEM_ENCRYPT for arm64 Realms, powerpc secure
guests, and s390 protected virtualization guests.
Dependency:
This series depends on the pKVM changes posted at:
https://lore.kernel.org/all/20260603110522.3331819-1-smostafa@google.com
Please merge this series only after the pKVM changes above are merged.
Otherwise pKVM will be broken.
Changes since v7:
https://lore.kernel.org/all/20260701054926.825925-1-aneesh.kumar@kernel.org
* Rebased onto dma-mapping-for-next
* Prepared the series on top of the prerequisite pKVM changes, resolving
conflicts so it can be applied directly to the pKVM topic branch once ready.
https://git.gitlab.arm.com/linux-arm/linux-cca/-/commits/scratch/pkvm/testing
* Added comments documenting possible follow-up improvements for CC_SHARED
atomic pools and physical-address-based pool freeing.
* Retained virtual-address-based pool freeing when
CONFIG_DMA_DIRECT_REMAP is disabled.
* Applied pgprot_decrypted() only when expanding a CC_SHARED atomic pool.
Changes since v6:
https://lore.kernel.org/all/20260604083959.1265923-1-aneesh.kumar@kernel.org
* Rebase onto the latest kernel.
* Add __DMA_ATTR_ALLOC_CC_SHARED for allocation paths. DMA_ATTR_CC_SHARED
is now used to describe the requested DMA mapping address attribute,
while __DMA_ATTR_ALLOC_CC_SHARED is used internally when allocating
shared/decrypted backing pages.
* Report CC_ATTR_GUEST_MEM_ENCRYPT for arm64 Realms, powerpc secure
guests, and s390 protected virtualization guests.
* Add CC_ATTR_HOST_MEM_ENCRYPT and swiotlb=force fixes.
Changes since v5:
https://lore.kernel.org/all/20260522042815.370873-1-aneesh.kumar@kernel.org
* Add Tested-by
* Drop the pKVM patch, which has now been posted separately:
https://lore.kernel.org/all/20260603110522.3331819-1-smostafa@google.com
* Remove the DO_NOT_MERGE tag from the s390 change.
* Add a patch to drop the SWIOTLB_FORCE flag.
* Rebase onto the latest kernel.
Changes since v4:
https://lore.kernel.org/all/20260512090408.794195-1-aneesh.kumar@kernel.org
* Add new patches based on Sashiko review:
swiotlb: Preserve allocation virtual address for dynamic pools
dma: free atomic pool pages by physical address
dma: swiotlb: handle set_memory_decrypted() failures
dma: swiotlb: free dynamic pools from process context
iommu/dma: Check atomic pool allocation result directly
* Include pKVM and s390 changes as dependent patches. These are not yet
ready to merge and are waiting for subsystem testing feedback.
* Drop the AMD GART patch because it requires wider testing.
* Update swiotlb_tbl_map_single() to take attrs by reference.
* Switch swiotlb_free() to use rcu_work.
* Avoid calling swiotlb_find_pool() multiple times in the free path.
* Make DMA_ATTR_MMIO imply DMA_ATTR_CC_SHARED for devices requiring unencrypted DMA.
Changes from v3:
https://lore.kernel.org/all/20260427055509.898190-1-aneesh.kumar@kernel.org
* Handle DMA_ATTR_MMIO correctly in dma_direct_map_phys()
* Address most of sashiko review
* Rebase to latest kernel
* drop SWIOTLB_FORCE for s390 and powerpc secure guest.
Changes from v2:
https://lore.kernel.org/all/20260420061415.3650870-1-aneesh.kumar@kernel.org
* pass attrs to dma_capable() and update direct, swiotlb, Xen swiotlb, and
x86 GART paths so the capability checks see the DMA address attr value
DMA_ATTR_CC_SHARED.
* rework dma_direct_map_phys() so DMA_ATTR_CC_SHARED selects
phys_to_dma_unencrypted() while the default path uses
phys_to_dma_encrypted(), with swiotlb fallback when the requested
shared/private state cannot be satisfied by a direct DMA address.
* stop relying on SWIOTLB_FORCE for arm64 and x86 CC guest DMA mappings;
swiotlb is still enabled there, but shared mappings is now selected
through the generic dma_direct_map_phys()/dma_capable() decision instead
of a global force-bounce flag.
Changes from v1:
https://lore.kernel.org/all/20260417085900.3062416-1-aneesh.kumar@kernel.org
* rebased to latest kernel (change from DMA_ATTR_CC_DECRYPTED -> DMA_ATTR_CC_SHARED)
* update the alloc path so DMA_ATTR_CC_SHARED is not a caller-visible attribute.
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Will Deacon <will@kernel.org>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Steven Price <steven.price@arm.com>
Cc: Suzuki K Poulose <Suzuki.Poulose@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Jiri Pirko <jiri@resnulli.us>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: Mostafa Saleh <smostafa@google.com>
Cc: Petr Tesarik <ptesarik@suse.com>
Cc: Alexey Kardashevskiy <aik@amd.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Xu Yilun <yilun.xu@linux.intel.com>
Cc: linuxppc-dev@lists.ozlabs.org
Cc: linux-s390@vger.kernel.org
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: "Christophe Leroy (CS GROUP)" <chleroy@kernel.org>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: x86@kernel.org
Aneesh Kumar K.V (Arm) (23):
dma-direct: return struct page from dma_direct_alloc_from_pool()
dma-pool: fix page leak in atomic_pool_expand() cleanup
iommu/dma: Check atomic pool allocation result directly
dma: free atomic pool pages by physical address
swiotlb: Preserve allocation virtual address for dynamic pools
s390: Expose protected virtualization through cc_platform_has()
dma-direct: swiotlb: handle swiotlb alloc/free outside
__dma_direct_alloc_pages
coco: arm64: s390: powerpc: Mark secure guests with
CC_ATTR_GUEST_MEM_ENCRYPT
dma-mapping: Add internal shared allocation attribute
dma-direct: use __DMA_ATTR_ALLOC_CC_SHARED in alloc/free paths
dma-pool: track decrypted atomic pools and select them via attrs
dma: swiotlb: pass mapping attributes by reference
dma: swiotlb: track pool encryption state and honor DMA_ATTR_CC_SHARED
dma-mapping: make dma_pgprot() honor __DMA_ATTR_ALLOC_CC_SHARED
dma-direct: pass attrs to dma_capable() for DMA_ATTR_CC_SHARED checks
dma-direct: Move dma_direct_map_phys() to dma/direct.c
dma-direct: make dma_direct_map_phys() honor DMA_ATTR_CC_SHARED
dma-direct: set decrypted flag for remapped DMA allocations
dma-direct: select DMA address encoding from
__DMA_ATTR_ALLOC_CC_SHARED
dma-direct: rename ret to cpu_addr in alloc helpers
dma: swiotlb: free dynamic pools from process context
dma: swiotlb: handle set_memory_decrypted() failures
swiotlb: remove unused SWIOTLB_FORCE flag
Documentation/core-api/dma-attributes.rst | 29 ++
arch/arm64/mm/init.c | 5 +-
arch/powerpc/platforms/pseries/cc_platform.c | 1 +
arch/powerpc/platforms/pseries/svm.c | 2 +-
arch/s390/Kconfig | 1 +
arch/s390/mm/init.c | 17 +-
arch/x86/kernel/amd_gart_64.c | 30 +-
arch/x86/kernel/pci-dma.c | 4 +-
drivers/iommu/dma-iommu.c | 20 +-
drivers/xen/swiotlb-xen.c | 8 +-
include/linux/dma-direct.h | 20 +-
include/linux/dma-map-ops.h | 3 +-
include/linux/dma-mapping.h | 8 +
include/linux/swiotlb.h | 25 +-
include/trace/events/dma.h | 3 +-
kernel/dma/direct.c | 326 ++++++++++++++-----
kernel/dma/direct.h | 56 +---
kernel/dma/mapping.c | 25 +-
kernel/dma/pool.c | 237 ++++++++++----
kernel/dma/swiotlb.c | 292 +++++++++++++----
20 files changed, 810 insertions(+), 302 deletions(-)
--
2.43.0
^ permalink raw reply
* Re: [PATCH 6/7] platform/raspberrypi: Remove redundant dev_err()
From: Stefan Wahren @ 2026-07-17 17:58 UTC (permalink / raw)
To: Pan Chuang, Florian Fainelli,
Broadcom internal kernel review list, Greg Kroah-Hartman,
Jai Luthra, Laurent Pinchart, Phil Elwell, Kees Cook,
moderated list:BROADCOM BCM2711/BCM2835 ARM ARCHITECTURE,
moderated list:BROADCOM BCM2711/BCM2835 ARM ARCHITECTURE,
open list
In-Reply-To: <20260717112103.213017-7-panchuang@vivo.com>
Am 17.07.26 um 13:20 schrieb Pan Chuang:
> Since commit 55b48e23f5c4 ("genirq/devres: Add error handling in
> devm_request_*_irq()"), devm_request_irq() automatically logs
> detailed error messages on failure. Remove the now-redundant
> driver-specific dev_err() calls.
>
> Signed-off-by: Pan Chuang <panchuang@vivo.com>
Reviewed-by: Stefan Wahren <wahrenst@gmx.net>
> ---
> drivers/platform/raspberrypi/vchiq-interface/vchiq_arm.c | 4 +---
> 1 file changed, 1 insertion(+), 3 deletions(-)
>
> diff --git a/drivers/platform/raspberrypi/vchiq-interface/vchiq_arm.c b/drivers/platform/raspberrypi/vchiq-interface/vchiq_arm.c
> index e78641703252..9490ad88df0f 100644
> --- a/drivers/platform/raspberrypi/vchiq-interface/vchiq_arm.c
> +++ b/drivers/platform/raspberrypi/vchiq-interface/vchiq_arm.c
> @@ -257,10 +257,8 @@ static int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state
>
> err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
> "VCHIQ doorbell", state);
> - if (err) {
> - dev_err(dev, "failed to register irq=%d\n", irq);
> + if (err)
> return err;
> - }
>
> /* Send the base address of the slots to VideoCore */
> channelbase = slot_phys;
^ permalink raw reply
* Re: [PATCH v2] arm64: syscall: Ensure saved x0 is kept in-sync with tracer updates
From: Will Deacon @ 2026-07-17 17:54 UTC (permalink / raw)
To: linux-arm-kernel
Cc: catalin.marinas, kernel-team, linux-kernel, Kees Cook,
Jinjie Ruan, Mark Rutland, Yiqi Sun
In-Reply-To: <178420908877.33796.2351583216856259313.b4-ty@kernel.org>
On Thu, Jul 16, 2026 at 05:48:01PM +0100, Will Deacon wrote:
> On Thu, 16 Jul 2026 13:06:39 +0100, Will Deacon wrote:
> > When seccomp support was originally added to arm64 in a1ae65b21941
> > ("arm64: add seccomp support"), seccomp was erroneously called _before_
> > the ptrace syscall-enter-stop and therefore the tracer could trivially
> > manipulate the syscall register state after the seccomp check had
> > passed. This was subsequently fixed in a5cd110cb836 ("arm64/ptrace: run
> > seccomp after ptrace") by moving the seccomp check after the tracer has
> > run. Unfortunately, a decade later, that fix has been reported to be
> > incomplete.
> >
> > [...]
>
> Applied to arm64 (for-next/fixes), thanks!
>
> [1/1] arm64: syscall: Ensure saved x0 is kept in-sync with tracer updates
> https://git.kernel.org/arm64/c/e057b9477232
Bah, I've had to revert this. I think Sashiko makes a good point here
that the seccomp interaction is still broken when the filter is
re-evaluated after the tracer stop, because that all happens inside
secure_computing() so we don't get a chance to update 'orig_x0':
https://sashiko.dev/#/patchset/20260716120640.6590-1-will@kernel.org
I've got a v3 that takes a different approach, so I'll send that out
shortly. Jinjie, thanks for sending the selftests, but maybe we can
extend them to cover the loophole above as wel?
Will
^ permalink raw reply
* [PATCH v2 3/3] mtd: rawnand: sunxi: fix H6/H616 controller timings
From: James Hilliard @ 2026-07-17 17:33 UTC (permalink / raw)
To: Miquel Raynal, Richard Weinberger, Vignesh Raghavendra,
Chen-Yu Tsai, Jernej Skrabec, Samuel Holland, Richard Genoud
Cc: linux-mtd, linux-arm-kernel, linux-sunxi, linux-kernel,
James Hilliard, stable
In-Reply-To: <20260717-submit-sunxi-nand-h6-h616-timings-v2-0-225515107640@gmail.com>
The NAND timing calculation assumes that command and address setup and
hold intervals T1-T4, T7 and T11 each take one controller clock. It also
uses the original A10 delay encodings for tWB, tADL, tWHR and tRHW.
The H6/H616 NDFC defines the setup and hold intervals as two internal
clock cycles and uses different delay encodings. Add the H616 timing
characteristics and select them through the controller capability data so
the clock solver and timing fields match the hardware.
Fixes: 88fd4e4deae8 ("mtd: rawnand: sunxi: Add support for H616 nand controller")
Cc: stable@vger.kernel.org
Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
drivers/mtd/nand/raw/sunxi_nand.c | 49 ++++++++++++++++++++++++++++-----------
1 file changed, 36 insertions(+), 13 deletions(-)
diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c
index 791f495f6e82..3d91533a9f13 100644
--- a/drivers/mtd/nand/raw/sunxi_nand.c
+++ b/drivers/mtd/nand/raw/sunxi_nand.c
@@ -241,6 +241,8 @@ struct sunxi_nand_hw_ecc {
/* Delay arrays contain internal NDFC clock cycles for field values 0 to 3. */
struct sunxi_nfc_timings {
+ /* Internal clock cycles used by T1-T4, T7 and T11. */
+ u8 setup_cycles;
s32 tWB[SUNXI_NFC_TIMING_STEPS];
s32 tADL[SUNXI_NFC_TIMING_STEPS];
s32 tWHR[SUNXI_NFC_TIMING_STEPS];
@@ -1680,12 +1682,21 @@ static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
}
static const struct sunxi_nfc_timings sun4i_a10_nfc_timings = {
+ .setup_cycles = 1,
.tWB = { 6, 12, 16, 20 },
.tADL = { 7, 15, 23, 31 },
.tWHR = { 7, 15, 23, 31 },
.tRHW = { 4, 8, 12, 20 },
};
+static const struct sunxi_nfc_timings sun50i_h616_nfc_timings = {
+ .setup_cycles = 2,
+ .tWB = { 28, 44, 60, 76 },
+ .tADL = { 0, 12, 28, 44 },
+ .tWHR = { 0, 12, 28, 44 },
+ .tRHW = { 8, 24, 40, 56 },
+};
+
static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
u32 clk_period)
{
@@ -1720,20 +1731,28 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
return -ENOTSUPP;
/* T1 <=> tCLS */
- if (timings->tCLS_min > min_clk_period)
- min_clk_period = timings->tCLS_min;
+ if (timings->tCLS_min >
+ min_clk_period * nfc_timings->setup_cycles)
+ min_clk_period = DIV_ROUND_UP(timings->tCLS_min,
+ nfc_timings->setup_cycles);
/* T2 <=> tCLH */
- if (timings->tCLH_min > min_clk_period)
- min_clk_period = timings->tCLH_min;
+ if (timings->tCLH_min >
+ min_clk_period * nfc_timings->setup_cycles)
+ min_clk_period = DIV_ROUND_UP(timings->tCLH_min,
+ nfc_timings->setup_cycles);
/* T3 <=> tCS */
- if (timings->tCS_min > min_clk_period)
- min_clk_period = timings->tCS_min;
+ if (timings->tCS_min >
+ min_clk_period * nfc_timings->setup_cycles)
+ min_clk_period = DIV_ROUND_UP(timings->tCS_min,
+ nfc_timings->setup_cycles);
/* T4 <=> tCH */
- if (timings->tCH_min > min_clk_period)
- min_clk_period = timings->tCH_min;
+ if (timings->tCH_min >
+ min_clk_period * nfc_timings->setup_cycles)
+ min_clk_period = DIV_ROUND_UP(timings->tCH_min,
+ nfc_timings->setup_cycles);
/* T5 <=> tWP */
if (timings->tWP_min > min_clk_period)
@@ -1744,8 +1763,10 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
min_clk_period = timings->tWH_min;
/* T7 <=> tALS */
- if (timings->tALS_min > min_clk_period)
- min_clk_period = timings->tALS_min;
+ if (timings->tALS_min >
+ min_clk_period * nfc_timings->setup_cycles)
+ min_clk_period = DIV_ROUND_UP(timings->tALS_min,
+ nfc_timings->setup_cycles);
/* T8 <=> tDS */
if (timings->tDS_min > min_clk_period)
@@ -1760,8 +1781,10 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
/* T11 <=> tALH */
- if (timings->tALH_min > min_clk_period)
- min_clk_period = timings->tALH_min;
+ if (timings->tALH_min >
+ min_clk_period * nfc_timings->setup_cycles)
+ min_clk_period = DIV_ROUND_UP(timings->tALH_min,
+ nfc_timings->setup_cycles);
/* T12 <=> tRP */
if (timings->tRP_min > min_clk_period)
@@ -2670,7 +2693,7 @@ static const struct sunxi_nfc_caps sunxi_nfc_h616_caps = {
.nuser_data_tab = ARRAY_SIZE(sunxi_user_data_len_h6),
.max_ecc_steps = 32,
.sram_size = 8192,
- .timings = &sun4i_a10_nfc_timings,
+ .timings = &sun50i_h616_nfc_timings,
};
static const struct of_device_id sunxi_nfc_ids[] = {
--
2.53.0
^ permalink raw reply related
* [PATCH v2 1/3] mtd: rawnand: sunxi: group controller delay tables
From: James Hilliard @ 2026-07-17 17:33 UTC (permalink / raw)
To: Miquel Raynal, Richard Weinberger, Vignesh Raghavendra,
Chen-Yu Tsai, Jernej Skrabec, Samuel Holland, Richard Genoud
Cc: linux-mtd, linux-arm-kernel, linux-sunxi, linux-kernel,
James Hilliard, stable
In-Reply-To: <20260717-submit-sunxi-nand-h6-h616-timings-v2-0-225515107640@gmail.com>
The tWB and tRHW timing field encodings are controller properties, but
they currently live in standalone lookup tables.
Group them in a timing descriptor selected through the controller
capability data. Point every existing controller at the legacy values so
this is a pure preparation change.
Cc: stable@vger.kernel.org
Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
drivers/mtd/nand/raw/sunxi_nand.c | 36 ++++++++++++++++++++++++++++--------
1 file changed, 28 insertions(+), 8 deletions(-)
diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c
index 02647565c8ba..d12cbb3c813c 100644
--- a/drivers/mtd/nand/raw/sunxi_nand.c
+++ b/drivers/mtd/nand/raw/sunxi_nand.c
@@ -237,6 +237,14 @@ struct sunxi_nand_hw_ecc {
u32 ecc_ctl;
};
+#define SUNXI_NFC_TIMING_STEPS 4
+
+/* Delay arrays contain internal NDFC clock cycles for field values 0 to 3. */
+struct sunxi_nfc_timings {
+ s32 tWB[SUNXI_NFC_TIMING_STEPS];
+ s32 tRHW[SUNXI_NFC_TIMING_STEPS];
+};
+
/**
* struct sunxi_nand_chip - stores NAND chip device related information
*
@@ -301,6 +309,7 @@ static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
* bytes to write
* @nuser_data_tab: Size of @user_data_len_tab
* @sram_size: Size of the NAND controller SRAM
+ * @timings: Controller timing characteristics
*/
struct sunxi_nfc_caps {
bool has_mdma;
@@ -327,6 +336,7 @@ struct sunxi_nfc_caps {
unsigned int nuser_data_tab;
unsigned int max_ecc_steps;
int sram_size;
+ const struct sunxi_nfc_timings *timings;
};
/**
@@ -1667,8 +1677,10 @@ static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
return nand_prog_page_end_op(nand);
}
-static const s32 tWB_lut[] = {6, 12, 16, 20};
-static const s32 tRHW_lut[] = {4, 8, 12, 20};
+static const struct sunxi_nfc_timings sun4i_a10_nfc_timings = {
+ .tWB = { 6, 12, 16, 20 },
+ .tRHW = { 4, 8, 12, 20 },
+};
static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
u32 clk_period)
@@ -1693,6 +1705,7 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
{
struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
struct sunxi_nfc *nfc = to_sunxi_nfc(sunxi_nand->nand.controller);
+ const struct sunxi_nfc_timings *nfc_timings = nfc->caps->timings;
const struct nand_sdr_timings *timings;
u32 min_clk_period = 0;
s32 tWB, tADL, tWHR, tRHW, tCAD;
@@ -1763,8 +1776,10 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
/* T16 - T19 + tCAD */
- if (timings->tWB_max > (min_clk_period * 20))
- min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
+ if (timings->tWB_max >
+ (min_clk_period * nfc_timings->tWB[SUNXI_NFC_TIMING_STEPS - 1]))
+ min_clk_period = DIV_ROUND_UP(timings->tWB_max,
+ nfc_timings->tWB[SUNXI_NFC_TIMING_STEPS - 1]);
if (timings->tADL_min > (min_clk_period * 32))
min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
@@ -1772,8 +1787,10 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
if (timings->tWHR_min > (min_clk_period * 32))
min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
- if (timings->tRHW_min > (min_clk_period * 20))
- min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
+ if (timings->tRHW_min >
+ (min_clk_period * nfc_timings->tRHW[SUNXI_NFC_TIMING_STEPS - 1]))
+ min_clk_period = DIV_ROUND_UP(timings->tRHW_min,
+ nfc_timings->tRHW[SUNXI_NFC_TIMING_STEPS - 1]);
/*
* In non-EDO, tREA should be less than tRP to guarantee that the
@@ -1789,7 +1806,7 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)
min_clk_period = timings->tREA_max;
- tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
+ tWB = sunxi_nand_lookup_timing(nfc_timings->tWB, timings->tWB_max,
min_clk_period);
if (tWB < 0) {
dev_err(nfc->dev, "unsupported tWB\n");
@@ -1808,7 +1825,7 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
return -EINVAL;
}
- tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
+ tRHW = sunxi_nand_lookup_timing(nfc_timings->tRHW, timings->tRHW_min,
min_clk_period);
if (tRHW < 0) {
dev_err(nfc->dev, "unsupported tRHW\n");
@@ -2595,6 +2612,7 @@ static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = {
.nstrengths = ARRAY_SIZE(sunxi_ecc_strengths_a10),
.max_ecc_steps = 16,
.sram_size = 1024,
+ .timings = &sun4i_a10_nfc_timings,
};
static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = {
@@ -2617,6 +2635,7 @@ static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = {
.nstrengths = ARRAY_SIZE(sunxi_ecc_strengths_a10),
.max_ecc_steps = 16,
.sram_size = 1024,
+ .timings = &sun4i_a10_nfc_timings,
};
static const struct sunxi_nfc_caps sunxi_nfc_h616_caps = {
@@ -2641,6 +2660,7 @@ static const struct sunxi_nfc_caps sunxi_nfc_h616_caps = {
.nuser_data_tab = ARRAY_SIZE(sunxi_user_data_len_h6),
.max_ecc_steps = 32,
.sram_size = 8192,
+ .timings = &sun4i_a10_nfc_timings,
};
static const struct of_device_id sunxi_nfc_ids[] = {
--
2.53.0
^ permalink raw reply related
* [PATCH v2 2/3] mtd: rawnand: sunxi: describe tADL and tWHR delays
From: James Hilliard @ 2026-07-17 17:33 UTC (permalink / raw)
To: Miquel Raynal, Richard Weinberger, Vignesh Raghavendra,
Chen-Yu Tsai, Jernej Skrabec, Samuel Holland, Richard Genoud
Cc: linux-mtd, linux-arm-kernel, linux-sunxi, linux-kernel,
James Hilliard, stable
In-Reply-To: <20260717-submit-sunxi-nand-h6-h616-timings-v2-0-225515107640@gmail.com>
The tADL and tWHR timing fields use four encoded delays, but the driver
currently derives their values with a shift. This hides the actual
controller timing characteristics and lets the clock solver select a
32-cycle delay that the fields cannot encode.
Describe the legacy 7, 15, 23 and 31 cycle thresholds explicitly and use
the tables for both clock selection and field lookup. This prepares the
driver for controllers with different encodings.
Cc: stable@vger.kernel.org
Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
drivers/mtd/nand/raw/sunxi_nand.c | 30 ++++++++++++++++++++----------
1 file changed, 20 insertions(+), 10 deletions(-)
diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c
index d12cbb3c813c..791f495f6e82 100644
--- a/drivers/mtd/nand/raw/sunxi_nand.c
+++ b/drivers/mtd/nand/raw/sunxi_nand.c
@@ -242,6 +242,8 @@ struct sunxi_nand_hw_ecc {
/* Delay arrays contain internal NDFC clock cycles for field values 0 to 3. */
struct sunxi_nfc_timings {
s32 tWB[SUNXI_NFC_TIMING_STEPS];
+ s32 tADL[SUNXI_NFC_TIMING_STEPS];
+ s32 tWHR[SUNXI_NFC_TIMING_STEPS];
s32 tRHW[SUNXI_NFC_TIMING_STEPS];
};
@@ -1679,6 +1681,8 @@ static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
static const struct sunxi_nfc_timings sun4i_a10_nfc_timings = {
.tWB = { 6, 12, 16, 20 },
+ .tADL = { 7, 15, 23, 31 },
+ .tWHR = { 7, 15, 23, 31 },
.tRHW = { 4, 8, 12, 20 },
};
@@ -1781,11 +1785,15 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
min_clk_period = DIV_ROUND_UP(timings->tWB_max,
nfc_timings->tWB[SUNXI_NFC_TIMING_STEPS - 1]);
- if (timings->tADL_min > (min_clk_period * 32))
- min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
+ if (timings->tADL_min >
+ (min_clk_period * nfc_timings->tADL[SUNXI_NFC_TIMING_STEPS - 1]))
+ min_clk_period = DIV_ROUND_UP(timings->tADL_min,
+ nfc_timings->tADL[SUNXI_NFC_TIMING_STEPS - 1]);
- if (timings->tWHR_min > (min_clk_period * 32))
- min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
+ if (timings->tWHR_min >
+ (min_clk_period * nfc_timings->tWHR[SUNXI_NFC_TIMING_STEPS - 1]))
+ min_clk_period = DIV_ROUND_UP(timings->tWHR_min,
+ nfc_timings->tWHR[SUNXI_NFC_TIMING_STEPS - 1]);
if (timings->tRHW_min >
(min_clk_period * nfc_timings->tRHW[SUNXI_NFC_TIMING_STEPS - 1]))
@@ -1813,16 +1821,18 @@ static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
return tWB;
}
- tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
- if (tADL > 3) {
+ tADL = sunxi_nand_lookup_timing(nfc_timings->tADL,
+ timings->tADL_min, min_clk_period);
+ if (tADL < 0) {
dev_err(nfc->dev, "unsupported tADL\n");
- return -EINVAL;
+ return tADL;
}
- tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
- if (tWHR > 3) {
+ tWHR = sunxi_nand_lookup_timing(nfc_timings->tWHR,
+ timings->tWHR_min, min_clk_period);
+ if (tWHR < 0) {
dev_err(nfc->dev, "unsupported tWHR\n");
- return -EINVAL;
+ return tWHR;
}
tRHW = sunxi_nand_lookup_timing(nfc_timings->tRHW, timings->tRHW_min,
--
2.53.0
^ permalink raw reply related
* [PATCH v2 0/3] mtd: rawnand: sunxi: fix H6/H616 controller timings
From: James Hilliard @ 2026-07-17 17:33 UTC (permalink / raw)
To: Miquel Raynal, Richard Weinberger, Vignesh Raghavendra,
Chen-Yu Tsai, Jernej Skrabec, Samuel Holland, Richard Genoud
Cc: linux-mtd, linux-arm-kernel, linux-sunxi, linux-kernel,
James Hilliard, stable
The sunxi NAND timing solver currently applies the original A10 NDFC
timing characteristics to every supported controller. The H6/H616 NDFC
uses two internal cycles for several setup and hold intervals and has
different delay-field encodings, so the resulting clock and timing
register values do not match the hardware.
Prepare for per-controller timing data in two behaviorally focused
patches, then add and select the H616 characteristics in the final fix.
Legacy controllers retain their existing setup-cycle behavior and delay
encodings.
Tested on an H616 board with Toshiba TC58NVG1S3H NAND.
Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
Changes v1 -> v2:
- Split legacy timing setup from the H616 fix (suggested by Miquel)
- Move tADL/tWHR encodings to a separate patch (suggested by Miquel)
- Keep the timing lookup helper unchanged (suggested by Miquel)
- Drop the minimum-period helper (suggested by Miquel)
- Align H616 timing-data and capability names (suggested by Miquel)
- Add Cc: stable to all three patches (suggested by Miquel)
---
James Hilliard (3):
mtd: rawnand: sunxi: group controller delay tables
mtd: rawnand: sunxi: describe tADL and tWHR delays
mtd: rawnand: sunxi: fix H6/H616 controller timings
drivers/mtd/nand/raw/sunxi_nand.c | 119 +++++++++++++++++++++++++++-----------
1 file changed, 86 insertions(+), 33 deletions(-)
---
base-commit: 6f5156d7a31a8c3b0f34af4675c9299c8f877cbe
change-id: 20260717-submit-sunxi-nand-h6-h616-timings-794a2994267d
Best regards,
--
James Hilliard <james.hilliard1@gmail.com>
^ permalink raw reply
* [PATCH mm-hotfixes v5 5/5] arm64: remove redundant concurrent ptdump UAF mitigation
From: Lorenzo Stoakes (ARM) @ 2026-07-17 17:30 UTC (permalink / raw)
To: Andrew Morton, Suren Baghdasaryan, Liam R. Howlett,
Vlastimil Babka, Shakeel Butt, David Hildenbrand, Mike Rapoport,
Michal Hocko, Uladzislau Rezki, Toshi Kani, Dave Hansen,
Andy Lutomirski, Peter Zijlstra, Thomas Gleixner, Ingo Molnar,
Borislav Petkov, x86, H. Peter Anvin, Kiryl Shutsemau,
Catalin Marinas, Will Deacon, Dev Jain, Ryan Roberts
Cc: David Carlier, ljs, linux-mm, linux-kernel, bpf, linux-arm-kernel,
Denis V. Lunev, stable
In-Reply-To: <20260717-series-vmap-race-fix-v5-0-606a0ac6d3e5@kernel.org>
This partially reverts commit fa93b45fd397 ("arm64: Enable vmalloc-huge
with ptdump"), retaining vmalloc-huge support but eliminating the now
redundant mitigation against a race between huge vmap page table freeing
and ptdump, as this issue has now been fixed at core.
We also simultaneously remove the arm64 if-deffery when acquiring the mmap
read lock upon vmap huge page table promotion as it is no longer required.
Note that this patch relies on the preceding vmalloc patch, and should not
be backported alone.
Fixes: fa93b45fd397 ("arm64: Enable vmalloc-huge with ptdump")
Cc: stable@vger.kernel.org
Reviewed-by: Dev Jain <dev.jain@arm.com>
Acked-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Acked-by: Kiryl Shutsemau (Meta) <kas@kernel.org>
Acked-by: Will Deacon <will@kernel.org>
Reviewed-by: David Hildenbrand (Arm) <david@kernel.org>
Signed-off-by: Lorenzo Stoakes (ARM) <ljs@kernel.org>
---
arch/arm64/include/asm/ptdump.h | 2 --
arch/arm64/mm/mmu.c | 43 ++++-------------------------------------
arch/arm64/mm/ptdump.c | 11 ++---------
mm/vmalloc.c | 15 +++-----------
4 files changed, 9 insertions(+), 62 deletions(-)
diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h
index 5b374a6ab34a..50a195eda8ed 100644
--- a/arch/arm64/include/asm/ptdump.h
+++ b/arch/arm64/include/asm/ptdump.h
@@ -7,8 +7,6 @@
#include <linux/ptdump.h>
-DECLARE_STATIC_KEY_FALSE(arm64_ptdump_lock_key);
-
#ifdef CONFIG_PTDUMP
#include <linux/mm_types.h>
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index a25d8beacc83..bd52fca6e872 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -49,8 +49,6 @@
#define NO_CONT_MAPPINGS BIT(1)
#define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */
-DEFINE_STATIC_KEY_FALSE(arm64_ptdump_lock_key);
-
u64 kimage_voffset __ro_after_init;
EXPORT_SYMBOL(kimage_voffset);
@@ -1864,8 +1862,7 @@ int pmd_clear_huge(pmd_t *pmdp)
return 1;
}
-static int __pmd_free_pte_page(pmd_t *pmdp, unsigned long addr,
- bool acquire_mmap_lock)
+int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
{
pte_t *table;
pmd_t pmd;
@@ -1877,25 +1874,13 @@ static int __pmd_free_pte_page(pmd_t *pmdp, unsigned long addr,
return 1;
}
- /* See comment in pud_free_pmd_page for static key logic */
table = pte_offset_kernel(pmdp, addr);
pmd_clear(pmdp);
__flush_tlb_kernel_pgtable(addr);
- if (static_branch_unlikely(&arm64_ptdump_lock_key) && acquire_mmap_lock) {
- mmap_read_lock(&init_mm);
- mmap_read_unlock(&init_mm);
- }
-
pte_free_kernel(NULL, table);
return 1;
}
-int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
-{
- /* If ptdump is walking the pagetables, acquire init_mm.mmap_lock */
- return __pmd_free_pte_page(pmdp, addr, /* acquire_mmap_lock = */ true);
-}
-
int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
{
pmd_t *table;
@@ -1911,36 +1896,16 @@ int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
}
table = pmd_offset(pudp, addr);
-
- /*
- * Our objective is to prevent ptdump from reading a PMD table which has
- * been freed. In this race, if pud_free_pmd_page observes the key on
- * (which got flipped by ptdump) then the mmap lock sequence here will,
- * as a result of the mmap write lock/unlock sequence in ptdump, give
- * us the correct synchronization. If not, this means that ptdump has
- * yet not started walking the pagetables - the sequence of barriers
- * issued by __flush_tlb_kernel_pgtable() guarantees that ptdump will
- * observe an empty PUD.
- */
- pud_clear(pudp);
- __flush_tlb_kernel_pgtable(addr);
- if (static_branch_unlikely(&arm64_ptdump_lock_key)) {
- mmap_read_lock(&init_mm);
- mmap_read_unlock(&init_mm);
- }
-
pmdp = table;
next = addr;
end = addr + PUD_SIZE;
do {
if (pmd_present(pmdp_get(pmdp)))
- /*
- * PMD has been isolated, so ptdump won't see it. No
- * need to acquire init_mm.mmap_lock.
- */
- __pmd_free_pte_page(pmdp, next, /* acquire_mmap_lock = */ false);
+ pmd_free_pte_page(pmdp, next);
} while (pmdp++, next += PMD_SIZE, next != end);
+ pud_clear(pudp);
+ __flush_tlb_kernel_pgtable(addr);
pmd_free(NULL, table);
return 1;
}
diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c
index 1c20144700d7..5a76c59b5ada 100644
--- a/arch/arm64/mm/ptdump.c
+++ b/arch/arm64/mm/ptdump.c
@@ -283,13 +283,6 @@ void note_page_flush(struct ptdump_state *pt_st)
note_page(pt_st, 0, -1, pte_val(pte_zero));
}
-static void arm64_ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm)
-{
- static_branch_inc(&arm64_ptdump_lock_key);
- ptdump_walk_pgd(st, mm, NULL);
- static_branch_dec(&arm64_ptdump_lock_key);
-}
-
void ptdump_walk(struct seq_file *s, struct ptdump_info *info)
{
unsigned long end = ~0UL;
@@ -318,7 +311,7 @@ void ptdump_walk(struct seq_file *s, struct ptdump_info *info)
}
};
- arm64_ptdump_walk_pgd(&st.ptdump, info->mm);
+ ptdump_walk_pgd(&st.ptdump, info->mm, NULL);
}
static void __init ptdump_initialize(void)
@@ -360,7 +353,7 @@ bool ptdump_check_wx(void)
}
};
- arm64_ptdump_walk_pgd(&st.ptdump, &init_mm);
+ ptdump_walk_pgd(&st.ptdump, &init_mm, NULL);
if (st.wx_pages || st.uxn_pages) {
pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found, %lu non-UXN pages found\n",
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index d5c4d2bb770b..f4fa227a8d7f 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -169,10 +169,7 @@ static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end,
* Concurrent read lock holders are safe: each exclusively owns
* the range it operates on and cannot reach this page table.
*/
-#ifndef CONFIG_ARM64
- scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm)
-#endif
- {
+ scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {
if (!pmd_free_pte_page(pmd, addr))
return 0;
return pmd_set_huge(pmd, phys_addr, prot);
@@ -229,10 +226,7 @@ static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end,
return pud_set_huge(pud, phys_addr, prot);
/* See comment in vmap_try_huge_pmd(). */
-#ifndef CONFIG_ARM64
- scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm)
-#endif
- {
+ scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {
if (!pud_free_pmd_page(pud, addr))
return 0;
return pud_set_huge(pud, phys_addr, prot);
@@ -289,10 +283,7 @@ static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end,
return p4d_set_huge(p4d, phys_addr, prot);
/* See comment in vmap_try_huge_pmd(). */
-#ifndef CONFIG_ARM64
- scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm)
-#endif
- {
+ scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) {
if (!p4d_free_pud_page(p4d, addr))
return 0;
return p4d_set_huge(p4d, phys_addr, prot);
--
2.55.0
^ permalink raw reply related
* [PATCH mm-hotfixes v5 4/5] mm/ptdump: always stabilise against page table freeing using init_mm
From: Lorenzo Stoakes (ARM) @ 2026-07-17 17:30 UTC (permalink / raw)
To: Andrew Morton, Suren Baghdasaryan, Liam R. Howlett,
Vlastimil Babka, Shakeel Butt, David Hildenbrand, Mike Rapoport,
Michal Hocko, Uladzislau Rezki, Toshi Kani, Dave Hansen,
Andy Lutomirski, Peter Zijlstra, Thomas Gleixner, Ingo Molnar,
Borislav Petkov, x86, H. Peter Anvin, Kiryl Shutsemau,
Catalin Marinas, Will Deacon, Dev Jain, Ryan Roberts
Cc: David Carlier, ljs, linux-mm, linux-kernel, bpf, linux-arm-kernel,
Denis V. Lunev, stable
In-Reply-To: <20260717-series-vmap-race-fix-v5-0-606a0ac6d3e5@kernel.org>
Previous commits have established the invariant that kernel page table
freeing is performed while an mmap read lock on init_mm is held, which
fixes races between ptdump and kernel page table freeing over init_mm.
However, x86 and arm64 can perform a ptdump over an mm other than init_mm
via ptdump_walk_pgd() and since kernel memory ranges are shared across
non-kernel mm's, this means that the race still exists for these cases.
Fix this by acquiring a nested mmap write lock for init_mm in
ptdump_walk_pgd().
This is safe as we take this after mmap write locking the mm, and nothing
acquires the init_mm lock first before locking an arbitrary mm, so no
deadlock is possible.
Also update walk_page_range_debug() to assert that init_mm is write locked,
add a comment explaining why and remove some redundant code, and eliminate
the unnecessary and confusing invocation of walk_kernel_page_table_range().
We can safely remove the non-NULL check for walk.mm, as the mmap lock
asserts would NULL pointer deref if it was (and of course no callers do
this).
The first point at which ptdump can race kernel page table freeing is
commit b6bdb7517c3d ("mm/vmalloc: add interfaces to free unmapped page
table"), so we target this in the Fixes tag.
Fixes: b6bdb7517c3d ("mm/vmalloc: add interfaces to free unmapped page table")
Cc: stable@vger.kernel.org
Reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Acked-by: David Hildenbrand (Arm) <david@kernel.org>
Reviewed-by: Kiryl Shutsemau <kas@kernel.org>
Signed-off-by: Lorenzo Stoakes (ARM) <ljs@kernel.org>
---
mm/pagewalk.c | 14 +++++++++-----
mm/ptdump.c | 7 +++++++
2 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index bbcfd68d0907..5d87c632a255 100644
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -702,12 +702,16 @@ int walk_page_range_debug(struct mm_struct *mm, unsigned long start,
* to account for page table freeing on vmap huge page mapping.
*/
mmap_assert_write_locked(mm);
+ /*
+ * x86, arm64 ptdump allow walks of efi mm's and x86 ptdump allows walks
+ * of arbitrary mm's.
+ *
+ * However, they both must also hold the init_mm lock to account for
+ * concurrent kernel page table freeing.
+ */
+ mmap_assert_write_locked(&init_mm);
- /* For convenience, we allow traversal of kernel mappings. */
- if (mm == &init_mm)
- return walk_kernel_page_table_range(start, end, ops,
- pgd, private);
- if (start >= end || !walk.mm)
+ if (start >= end)
return -EINVAL;
if (!check_ops_safe(ops))
return -EINVAL;
diff --git a/mm/ptdump.c b/mm/ptdump.c
index 973020000096..5851096e6f65 100644
--- a/mm/ptdump.c
+++ b/mm/ptdump.c
@@ -178,11 +178,18 @@ void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd)
get_online_mems();
mmap_write_lock(mm);
+ /* To stabilise kernel page tables we must hold the init_mm lock too. */
+ if (mm != &init_mm)
+ mmap_write_lock_nested(&init_mm, SINGLE_DEPTH_NESTING);
+
while (range->start != range->end) {
walk_page_range_debug(mm, range->start, range->end,
&ptdump_ops, pgd, st);
range++;
}
+
+ if (mm != &init_mm)
+ mmap_write_unlock(&init_mm);
mmap_write_unlock(mm);
put_online_mems();
--
2.55.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox