* [PATCH v5 16/20] iommu/dma: Check atomic pool allocation result directly
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
The non-blocking, non-coherent allocation path uses dma_alloc_from_pool(),
which returns the allocated page and fills cpu_addr only on success.
Do not rely on cpu_addr to detect allocation failure in this path. Check
the returned page directly before using it for the IOMMU mapping.
Fixes: 9420139f516d ("dma-pool: fix coherent pool allocations for IOMMU mappings")
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
drivers/iommu/dma-iommu.c | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 725c7adb0a8d..52c599f4472c 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1671,13 +1671,16 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
}
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
- !gfpflags_allow_blocking(gfp) && !coherent)
+ !gfpflags_allow_blocking(gfp) && !coherent) {
page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
gfp, attrs, NULL);
- else
+ if (!page)
+ return NULL;
+ } else {
cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
- if (!cpu_addr)
- return NULL;
+ if (!cpu_addr)
+ return NULL;
+ }
*handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot,
dev->coherent_dma_mask);
--
2.43.0
^ permalink raw reply related
* [PATCH v5 15/20] dma-direct: return struct page from dma_direct_alloc_from_pool()
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, stable
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Commit 5b138c534fda ("dma-direct: factor out a dma_direct_alloc_from_pool
helper") changed dma_direct_alloc_from_pool() to return the CPU address
from dma_alloc_from_pool(). That fits dma_direct_alloc(), but
dma_direct_alloc_pages() also uses the helper and expects a struct page *.
Fix this by making dma_direct_alloc_from_pool() return the struct page *
again, and pass the CPU address back through an out-parameter for the
dma_direct_alloc() caller.
Fixes: 5b138c534fda ("dma-direct: factor out a dma_direct_alloc_from_pool helper")
Cc: stable@vger.kernel.org
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 21 ++++++++++++---------
1 file changed, 12 insertions(+), 9 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 6b00c7f4a78b..907c6084c616 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -157,24 +157,24 @@ static bool dma_direct_use_pool(struct device *dev, gfp_t gfp)
return !gfpflags_allow_blocking(gfp) && !is_swiotlb_for_alloc(dev);
}
-static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
- dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
+static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size,
+ dma_addr_t *dma_handle, void **cpu_addr, gfp_t gfp,
+ unsigned long attrs)
{
struct page *page;
u64 phys_limit;
- void *ret;
if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)))
return NULL;
gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
- page = dma_alloc_from_pool(dev, size, &ret, gfp, attrs,
+ page = dma_alloc_from_pool(dev, size, cpu_addr, gfp, attrs,
dma_coherent_ok);
if (!page)
return NULL;
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
!!(attrs & DMA_ATTR_CC_SHARED));
- return ret;
+ return page;
}
static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,
@@ -270,9 +270,12 @@ void *dma_direct_alloc(struct device *dev, size_t size,
* the atomic pools instead if we aren't allowed block.
*/
if ((remap || (attrs & DMA_ATTR_CC_SHARED)) &&
- dma_direct_use_pool(dev, gfp))
- return dma_direct_alloc_from_pool(dev, size, dma_handle,
- gfp, attrs);
+ dma_direct_use_pool(dev, gfp)) {
+ page = dma_direct_alloc_from_pool(dev, size,
+ dma_handle, &cpu_addr,
+ gfp, attrs);
+ return page ? cpu_addr : NULL;
+ }
if (is_swiotlb_for_alloc(dev)) {
page = dma_direct_alloc_swiotlb(dev, size, attrs);
@@ -445,7 +448,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
if ((attrs & DMA_ATTR_CC_SHARED) && dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle,
- gfp, attrs);
+ &cpu_addr, gfp, attrs);
if (is_swiotlb_for_alloc(dev)) {
page = dma_direct_alloc_swiotlb(dev, size, attrs);
--
2.43.0
^ permalink raw reply related
* [PATCH v5 14/20] dma-direct: rename ret to cpu_addr in alloc helpers
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
ret in dma_direct_alloc() and dma_direct_alloc_pages() holds the returned
CPU mapping, not a generic return value. Rename it to cpu_addr and update
the remaining uses to match.
This makes the allocation paths easier to follow and keeps the local naming
consistent with what the variable actually represents.
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 31 +++++++++++++++----------------
1 file changed, 15 insertions(+), 16 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 6f3aff8448a0..6b00c7f4a78b 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -204,7 +204,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
bool mark_mem_decrypt = false;
bool allow_highmem = true;
struct page *page;
- void *ret;
+ void *cpu_addr;
/*
* DMA_ATTR_CC_SHARED is not a caller-visible dma_alloc_*()
@@ -318,34 +318,33 @@ void *dma_direct_alloc(struct device *dev, size_t size,
arch_dma_prep_coherent(page, size);
/* create a coherent mapping */
- ret = dma_common_contiguous_remap(page, size, prot,
- __builtin_return_address(0));
- if (!ret)
+ cpu_addr = dma_common_contiguous_remap(page, size, prot,
+ __builtin_return_address(0));
+ if (!cpu_addr)
goto out_encrypt_pages;
} else {
- ret = page_address(page);
+ cpu_addr = page_address(page);
}
- memset(ret, 0, size);
+ memset(cpu_addr, 0, size);
if (set_uncached) {
void *uncached_cpu_addr;
arch_dma_prep_coherent(page, size);
- uncached_cpu_addr = arch_dma_set_uncached(ret, size);
+ uncached_cpu_addr = arch_dma_set_uncached(cpu_addr, size);
if (IS_ERR(uncached_cpu_addr))
goto out_free_remap_pages;
- ret = uncached_cpu_addr;
+ cpu_addr = uncached_cpu_addr;
}
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
!!(attrs & DMA_ATTR_CC_SHARED));
- return ret;
-
+ return cpu_addr;
out_free_remap_pages:
if (remap)
- dma_common_free_remap(ret, size);
+ dma_common_free_remap(cpu_addr, size);
out_encrypt_pages:
if (mark_mem_decrypt &&
@@ -439,7 +438,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
{
unsigned long attrs = 0;
struct page *page;
- void *ret;
+ void *cpu_addr;
if (force_dma_unencrypted(dev))
attrs |= DMA_ATTR_CC_SHARED;
@@ -453,7 +452,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
if (!page)
return NULL;
- ret = page_address(page);
+ cpu_addr = page_address(page);
goto setup_page;
}
@@ -461,11 +460,11 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
if (!page)
return NULL;
- ret = page_address(page);
- if ((attrs & DMA_ATTR_CC_SHARED) && dma_set_decrypted(dev, ret, size))
+ cpu_addr = page_address(page);
+ if ((attrs & DMA_ATTR_CC_SHARED) && dma_set_decrypted(dev, cpu_addr, size))
goto out_leak_pages;
setup_page:
- memset(ret, 0, size);
+ memset(cpu_addr, 0, size);
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
!!(attrs & DMA_ATTR_CC_SHARED));
return page;
--
2.43.0
^ permalink raw reply related
* [PATCH v5 13/20] dma-pool: fix page leak in atomic_pool_expand() cleanup
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
atomic_pool_expand() frees the allocated pages from the remove_mapping
error path only when CONFIG_DMA_DIRECT_REMAP is enabled.
When CONFIG_DMA_DIRECT_REMAP is disabled, failures after page allocation,
such as gen_pool_add_virt(), jump to remove_mapping and return without
freeing the pages.
Move __free_pages(page, order) out of the CONFIG_DMA_DIRECT_REMAP block so
that cleanup paths always release the allocation.
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/pool.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c
index be78474a6c49..e7df8d279e75 100644
--- a/kernel/dma/pool.c
+++ b/kernel/dma/pool.c
@@ -146,9 +146,9 @@ static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_size,
#ifdef CONFIG_DMA_DIRECT_REMAP
dma_common_free_remap(addr, pool_size);
free_page:
+#endif
if (!leak_pages)
__free_pages(page, order);
-#endif
out:
return ret;
}
--
2.43.0
^ permalink raw reply related
* [PATCH v5 12/20] dma-direct: select DMA address encoding from DMA_ATTR_CC_SHARED
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Make the dma-direct helpers derive the DMA address encoding from
DMA_ATTR_CC_SHARED instead of implicitly relying on
force_dma_unencrypted() inside phys_to_dma_direct()
Pass an explicit unencrypted/decrypted state into phys_to_dma_direct(),
make the alloc paths return DMA addresses that match the requested buffer
encryption state. Also only call dma_set_decrypted() when
DMA_ATTR_CC_SHARED is actually set.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 42 +++++++++++++++++++++++++-----------------
1 file changed, 25 insertions(+), 17 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index d7eee77f95af..6f3aff8448a0 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -24,11 +24,11 @@
u64 zone_dma_limit __ro_after_init = DMA_BIT_MASK(24);
static inline dma_addr_t phys_to_dma_direct(struct device *dev,
- phys_addr_t phys)
+ phys_addr_t phys, bool unencrypted)
{
- if (force_dma_unencrypted(dev))
+ if (unencrypted)
return phys_to_dma_unencrypted(dev, phys);
- return phys_to_dma(dev, phys);
+ return phys_to_dma_encrypted(dev, phys);
}
static inline struct page *dma_direct_to_page(struct device *dev,
@@ -39,8 +39,9 @@ static inline struct page *dma_direct_to_page(struct device *dev,
u64 dma_direct_get_required_mask(struct device *dev)
{
+ bool require_decrypted = force_dma_unencrypted(dev);
phys_addr_t phys = (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT;
- u64 max_dma = phys_to_dma_direct(dev, phys);
+ u64 max_dma = phys_to_dma_direct(dev, phys, require_decrypted);
return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
}
@@ -69,7 +70,8 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit)
bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
{
- dma_addr_t dma_addr = phys_to_dma_direct(dev, phys);
+ bool require_decrypted = force_dma_unencrypted(dev);
+ dma_addr_t dma_addr = phys_to_dma_direct(dev, phys, require_decrypted);
if (dma_addr == DMA_MAPPING_ERROR)
return false;
@@ -79,17 +81,18 @@ bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size)
{
- if (!force_dma_unencrypted(dev))
- return 0;
- return set_memory_decrypted((unsigned long)vaddr, PFN_UP(size));
+ int ret;
+
+ ret = set_memory_decrypted((unsigned long)vaddr, PFN_UP(size));
+ if (ret)
+ pr_warn_ratelimited("leaking DMA memory that can't be decrypted\n");
+ return ret;
}
static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)
{
int ret;
- if (!force_dma_unencrypted(dev))
- return 0;
ret = set_memory_encrypted((unsigned long)vaddr, PFN_UP(size));
if (ret)
pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n");
@@ -169,7 +172,8 @@ static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
dma_coherent_ok);
if (!page)
return NULL;
- *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
+ *dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
+ !!(attrs & DMA_ATTR_CC_SHARED));
return ret;
}
@@ -185,9 +189,11 @@ static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,
/* remove any dirty cache lines on the kernel alias */
if (!PageHighMem(page))
arch_dma_prep_coherent(page, size);
-
- /* return the page pointer as the opaque cookie */
- *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
+ /*
+ * return the page pointer as the opaque cookie.
+ * Never used for unencrypted allocation
+ */
+ *dma_handle = phys_to_dma_encrypted(dev, page_to_phys(page));
return page;
}
@@ -332,7 +338,8 @@ void *dma_direct_alloc(struct device *dev, size_t size,
ret = uncached_cpu_addr;
}
- *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
+ *dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
+ !!(attrs & DMA_ATTR_CC_SHARED));
return ret;
@@ -455,11 +462,12 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
return NULL;
ret = page_address(page);
- if (dma_set_decrypted(dev, ret, size))
+ if ((attrs & DMA_ATTR_CC_SHARED) && dma_set_decrypted(dev, ret, size))
goto out_leak_pages;
setup_page:
memset(ret, 0, size);
- *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
+ *dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
+ !!(attrs & DMA_ATTR_CC_SHARED));
return page;
out_leak_pages:
return NULL;
--
2.43.0
^ permalink raw reply related
* [PATCH v5 11/20] dma-direct: set decrypted flag for remapped DMA allocations
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Devices that are DMA non-coherent and require a remap were skipping
dma_set_decrypted(), leaving DMA buffers encrypted even when the device
requires unencrypted access. Move the call after the if (remap) branch
so that both the direct and remapped allocation paths correctly mark the
allocation as decrypted (or fail cleanly) before use.
Fix dma_direct_alloc() and dma_direct_free() to apply set_memory_*() to
the linear-map alias of the backing pages instead of the remapped CPU
address. Also disallow highmem pages for DMA_ATTR_CC_SHARED, because
highmem buffers do not provide a usable linear-map address.
Fixes: f3c962226dbe ("dma-direct: clean up the remapping checks in dma_direct_alloc")
Tested-by: Jiri Pirko <jiri@nvidia.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 55 ++++++++++++++++++++++++++++++++++++---------
1 file changed, 44 insertions(+), 11 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 9e65c8432b6e..d7eee77f95af 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -196,6 +196,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
{
bool remap = false, set_uncached = false;
bool mark_mem_decrypt = false;
+ bool allow_highmem = true;
struct page *page;
void *ret;
@@ -214,6 +215,15 @@ void *dma_direct_alloc(struct device *dev, size_t size,
mark_mem_decrypt = true;
}
+ if (attrs & DMA_ATTR_CC_SHARED)
+ /*
+ * Unencrypted/shared DMA requires a linear-mapped buffer
+ * address to look up the PFN and set architecture-required PFN
+ * attributes. This is not possible with HighMem. Avoid HighMem
+ * allocation.
+ */
+ allow_highmem = false;
+
size = PAGE_ALIGN(size);
if (attrs & DMA_ATTR_NO_WARN)
gfp |= __GFP_NOWARN;
@@ -272,7 +282,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
}
/* we always manually zero the memory once we are done */
- page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);
+ page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, allow_highmem);
if (!page)
return NULL;
@@ -287,6 +297,14 @@ void *dma_direct_alloc(struct device *dev, size_t size,
set_uncached = false;
}
+ if (mark_mem_decrypt) {
+ void *lm_addr;
+
+ lm_addr = page_address(page);
+ if (set_memory_decrypted((unsigned long)lm_addr, PFN_UP(size)))
+ goto out_leak_pages;
+ }
+
if (remap) {
pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
@@ -297,29 +315,36 @@ void *dma_direct_alloc(struct device *dev, size_t size,
ret = dma_common_contiguous_remap(page, size, prot,
__builtin_return_address(0));
if (!ret)
- goto out_free_pages;
+ goto out_encrypt_pages;
} else {
ret = page_address(page);
- if (mark_mem_decrypt && dma_set_decrypted(dev, ret, size))
- goto out_leak_pages;
}
memset(ret, 0, size);
if (set_uncached) {
+ void *uncached_cpu_addr;
+
arch_dma_prep_coherent(page, size);
- ret = arch_dma_set_uncached(ret, size);
- if (IS_ERR(ret))
- goto out_encrypt_pages;
+ uncached_cpu_addr = arch_dma_set_uncached(ret, size);
+ if (IS_ERR(uncached_cpu_addr))
+ goto out_free_remap_pages;
+ ret = uncached_cpu_addr;
}
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
return ret;
+
+out_free_remap_pages:
+ if (remap)
+ dma_common_free_remap(ret, size);
+
out_encrypt_pages:
- if (mark_mem_decrypt && dma_set_encrypted(dev, page_address(page), size))
- return NULL;
-out_free_pages:
+ if (mark_mem_decrypt &&
+ dma_set_encrypted(dev, page_address(page), size))
+ goto out_leak_pages;
+
if (!swiotlb_free(dev, page, size))
dma_free_contiguous(dev, page, size);
return NULL;
@@ -384,8 +409,16 @@ void dma_direct_free(struct device *dev, size_t size,
} else {
if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
arch_dma_clear_uncached(cpu_addr, size);
- if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size))
+ }
+
+ if (mark_mem_encrypted) {
+ void *lm_addr;
+
+ lm_addr = phys_to_virt(phys);
+ if (set_memory_encrypted((unsigned long)lm_addr, PFN_UP(size))) {
+ pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n");
return;
+ }
}
if (swiotlb_pool)
--
2.43.0
^ permalink raw reply related
* [PATCH v5 10/20] dma-direct: make dma_direct_map_phys() honor DMA_ATTR_CC_SHARED
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Teach dma_direct_map_phys() to select the DMA address encoding based on
DMA_ATTR_CC_SHARED.
Use phys_to_dma_unencrypted() for decrypted mappings and
phys_to_dma_encrypted() otherwise. If a device requires unencrypted DMA
but the source physical address is still encrypted, force the mapping
through swiotlb so the DMA address and backing memory attributes remain
consistent.
Update the arm64, x86, s390 and powerpc secure-guest setup to not use
swiotlb force option
Tested-by: Jiri Pirko <jiri@nvidia.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
Changes from v3:
* Handle DMA_ATTR_MMIO
---
arch/arm64/mm/init.c | 4 +--
arch/powerpc/platforms/pseries/svm.c | 2 +-
arch/s390/mm/init.c | 2 +-
arch/x86/kernel/pci-dma.c | 4 +--
kernel/dma/direct.c | 4 ++-
kernel/dma/direct.h | 45 +++++++++++++++-------------
6 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index c1b223e7cc8e..a087ac5b15f7 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -340,10 +340,8 @@ void __init arch_mm_preinit(void)
unsigned int flags = SWIOTLB_VERBOSE;
bool swiotlb = max_pfn > PFN_DOWN(arm64_dma_phys_limit);
- if (is_realm_world()) {
+ if (is_realm_world())
swiotlb = true;
- flags |= SWIOTLB_FORCE;
- }
if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb) {
/*
diff --git a/arch/powerpc/platforms/pseries/svm.c b/arch/powerpc/platforms/pseries/svm.c
index 384c9dc1899a..7a403dbd35ee 100644
--- a/arch/powerpc/platforms/pseries/svm.c
+++ b/arch/powerpc/platforms/pseries/svm.c
@@ -29,7 +29,7 @@ static int __init init_svm(void)
* need to use the SWIOTLB buffer for DMA even if dma_capable() says
* otherwise.
*/
- ppc_swiotlb_flags |= SWIOTLB_ANY | SWIOTLB_FORCE;
+ ppc_swiotlb_flags |= SWIOTLB_ANY;
/* Share the SWIOTLB buffer with the host. */
swiotlb_update_mem_attributes();
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index ad3c6d92b801..581af1483c42 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -163,7 +163,7 @@ static void __init pv_init(void)
virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc);
/* make sure bounce buffers are shared */
- swiotlb_init(true, SWIOTLB_FORCE | SWIOTLB_VERBOSE);
+ swiotlb_init(true, SWIOTLB_VERBOSE);
swiotlb_update_mem_attributes();
}
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index 6267363e0189..75cf8f6ae8cd 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -59,10 +59,8 @@ static void __init pci_swiotlb_detect(void)
* bounce buffers as the hypervisor can't access arbitrary VM memory
* that is not explicitly shared with it.
*/
- if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
+ if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
x86_swiotlb_enable = true;
- x86_swiotlb_flags |= SWIOTLB_FORCE;
- }
}
#else
static inline void __init pci_swiotlb_detect(void)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 429791b2599a..9e65c8432b6e 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -702,8 +702,10 @@ size_t dma_direct_max_mapping_size(struct device *dev)
{
/* If SWIOTLB is active, use its maximum mapping size */
if (is_swiotlb_active(dev) &&
- (dma_addressing_limited(dev) || is_swiotlb_force_bounce(dev)))
+ (dma_addressing_limited(dev) || is_swiotlb_force_bounce(dev) ||
+ force_dma_unencrypted(dev)))
return swiotlb_max_mapping_size(dev);
+
return SIZE_MAX;
}
diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h
index e05dc7649366..f3fc28f352ba 100644
--- a/kernel/dma/direct.h
+++ b/kernel/dma/direct.h
@@ -88,37 +88,40 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev,
{
dma_addr_t dma_addr;
+ /*
+ * For a device requiring unencrypted DMA, MMIO memory is treated
+ * as shared by default.
+ */
+ if (force_dma_unencrypted(dev) && (attrs & DMA_ATTR_MMIO))
+ attrs |= DMA_ATTR_CC_SHARED;
+
if (is_swiotlb_force_bounce(dev)) {
- if (!(attrs & DMA_ATTR_CC_SHARED)) {
- if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
- return DMA_MAPPING_ERROR;
+ if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
+ return DMA_MAPPING_ERROR;
- return swiotlb_map(dev, phys, size, dir, attrs);
- }
- } else if (attrs & DMA_ATTR_CC_SHARED) {
- return DMA_MAPPING_ERROR;
+ return swiotlb_map(dev, phys, size, dir, attrs);
}
- if (attrs & DMA_ATTR_MMIO) {
- dma_addr = phys;
- if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs)))
- goto err_overflow;
- } else if (attrs & DMA_ATTR_CC_SHARED) {
+ if (attrs & DMA_ATTR_CC_SHARED)
dma_addr = phys_to_dma_unencrypted(dev, phys);
+ else
+ dma_addr = phys_to_dma_encrypted(dev, phys);
+
+ if (attrs & DMA_ATTR_MMIO) {
if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs)))
goto err_overflow;
- } else {
- dma_addr = phys_to_dma(dev, phys);
- if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) ||
- dma_kmalloc_needs_bounce(dev, size, dir)) {
- if (is_swiotlb_active(dev) &&
- !(attrs & DMA_ATTR_REQUIRE_COHERENT))
- return swiotlb_map(dev, phys, size, dir, attrs);
+ goto dma_mapped;
+ }
- goto err_overflow;
- }
+ if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) ||
+ dma_kmalloc_needs_bounce(dev, size, dir)) {
+ if (is_swiotlb_active(dev) &&
+ !(attrs & DMA_ATTR_REQUIRE_COHERENT))
+ return swiotlb_map(dev, phys, size, dir, attrs);
+ goto err_overflow;
}
+dma_mapped:
if (!dev_is_dma_coherent(dev) &&
!(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) {
arch_sync_dma_for_device(phys, size, dir);
--
2.43.0
^ permalink raw reply related
* [PATCH v5 09/20] dma-direct: pass attrs to dma_capable() for DMA_ATTR_CC_SHARED checks
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Teach dma_capable() about DMA_ATTR_CC_SHARED so the capability
check can reject encrypted DMA addresses for devices that require
unencrypted/shared DMA.
Also propagate DMA_ATTR_CC_SHARED in swiotlb_map() when the selected
SWIOTLB pool is decrypted so the capability check sees the correct DMA
address attribute.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
arch/x86/kernel/amd_gart_64.c | 30 ++++++++++++++++--------------
drivers/xen/swiotlb-xen.c | 6 +++---
include/linux/dma-direct.h | 10 +++++++++-
kernel/dma/direct.h | 6 +++---
kernel/dma/swiotlb.c | 2 +-
5 files changed, 32 insertions(+), 22 deletions(-)
diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c
index e8000a56732e..b5f1f031d45b 100644
--- a/arch/x86/kernel/amd_gart_64.c
+++ b/arch/x86/kernel/amd_gart_64.c
@@ -180,22 +180,23 @@ static void iommu_full(struct device *dev, size_t size, int dir)
}
static inline int
-need_iommu(struct device *dev, unsigned long addr, size_t size)
+need_iommu(struct device *dev, unsigned long addr, size_t size, unsigned long attrs)
{
- return force_iommu || !dma_capable(dev, addr, size, true);
+ return force_iommu || !dma_capable(dev, addr, size, true, attrs);
}
static inline int
-nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
+nonforced_iommu(struct device *dev, unsigned long addr, size_t size,
+ unsigned long attrs)
{
- return !dma_capable(dev, addr, size, true);
+ return !dma_capable(dev, addr, size, true, attrs);
}
/* Map a single continuous physical area into the IOMMU.
* Caller needs to check if the iommu is needed and flush.
*/
static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
- size_t size, int dir, unsigned long align_mask)
+ size_t size, int dir, unsigned long align_mask, unsigned long attrs)
{
unsigned long npages = iommu_num_pages(phys_mem, size, PAGE_SIZE);
unsigned long iommu_page;
@@ -206,7 +207,7 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
iommu_page = alloc_iommu(dev, npages, align_mask);
if (iommu_page == -1) {
- if (!nonforced_iommu(dev, phys_mem, size))
+ if (!nonforced_iommu(dev, phys_mem, size, attrs))
return phys_mem;
if (panic_on_overflow)
panic("dma_map_area overflow %lu bytes\n", size);
@@ -231,10 +232,10 @@ static dma_addr_t gart_map_phys(struct device *dev, phys_addr_t paddr,
if (unlikely(attrs & DMA_ATTR_MMIO))
return DMA_MAPPING_ERROR;
- if (!need_iommu(dev, paddr, size))
+ if (!need_iommu(dev, paddr, size, attrs))
return paddr;
- bus = dma_map_area(dev, paddr, size, dir, 0);
+ bus = dma_map_area(dev, paddr, size, dir, 0, attrs);
flush_gart();
return bus;
@@ -289,7 +290,7 @@ static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
/* Fallback for dma_map_sg in case of overflow */
static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
- int nents, int dir)
+ int nents, int dir, unsigned long attrs)
{
struct scatterlist *s;
int i;
@@ -301,8 +302,8 @@ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
for_each_sg(sg, s, nents, i) {
unsigned long addr = sg_phys(s);
- if (nonforced_iommu(dev, addr, s->length)) {
- addr = dma_map_area(dev, addr, s->length, dir, 0);
+ if (nonforced_iommu(dev, addr, s->length, attrs)) {
+ addr = dma_map_area(dev, addr, s->length, dir, 0, attrs);
if (addr == DMA_MAPPING_ERROR) {
if (i > 0)
gart_unmap_sg(dev, sg, i, dir, 0);
@@ -401,7 +402,7 @@ static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
s->dma_address = addr;
BUG_ON(s->length == 0);
- nextneed = need_iommu(dev, addr, s->length);
+ nextneed = need_iommu(dev, addr, s->length, attrs);
/* Handle the previous not yet processed entries */
if (i > start) {
@@ -449,7 +450,7 @@ static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
/* When it was forced or merged try again in a dumb way */
if (force_iommu || iommu_merge) {
- out = dma_map_sg_nonforce(dev, sg, nents, dir);
+ out = dma_map_sg_nonforce(dev, sg, nents, dir, attrs);
if (out > 0)
return out;
}
@@ -473,7 +474,8 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
return vaddr;
*dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size,
- DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);
+ DMA_BIDIRECTIONAL,
+ (1UL << get_order(size)) - 1, attrs);
flush_gart();
if (unlikely(*dma_addr == DMA_MAPPING_ERROR))
goto out_free;
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 8c4abe65cd49..e2538824ef52 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -212,7 +212,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
BUG_ON(dir == DMA_NONE);
if (attrs & DMA_ATTR_MMIO) {
- if (unlikely(!dma_capable(dev, phys, size, false))) {
+ if (unlikely(!dma_capable(dev, phys, size, false, attrs))) {
dev_err_once(
dev,
"DMA addr %pa+%zu overflow (mask %llx, bus limit %llx).\n",
@@ -231,7 +231,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
* we can safely return the device addr and not worry about bounce
* buffering it.
*/
- if (dma_capable(dev, dev_addr, size, true) &&
+ if (dma_capable(dev, dev_addr, size, true, attrs) &&
!dma_kmalloc_needs_bounce(dev, size, dir) &&
!range_straddles_page_boundary(phys, size) &&
!xen_arch_need_swiotlb(dev, phys, dev_addr) &&
@@ -253,7 +253,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
/*
* Ensure that the address returned is DMA'ble
*/
- if (unlikely(!dma_capable(dev, dev_addr, size, true))) {
+ if (unlikely(!dma_capable(dev, dev_addr, size, true, attrs))) {
__swiotlb_tbl_unmap_single(dev, map, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC,
swiotlb_find_pool(dev, map));
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index 94fad4e7c11e..daa31a1adf7b 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -135,12 +135,20 @@ static inline bool force_dma_unencrypted(struct device *dev)
#endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */
static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size,
- bool is_ram)
+ bool is_ram, unsigned long attrs)
{
dma_addr_t end = addr + size - 1;
if (addr == DMA_MAPPING_ERROR)
return false;
+ /*
+ * The DMA address was derived from encrypted RAM, but this device
+ * requires unencrypted DMA addresses. Treat it as not DMA-capable
+ * so the caller can fall back to a suitable SWIOTLB pool.
+ */
+ if (!(attrs & DMA_ATTR_CC_SHARED) && force_dma_unencrypted(dev))
+ return false;
+
if (is_ram && !IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) &&
min(addr, end) < phys_to_dma(dev, PFN_PHYS(min_low_pfn)))
return false;
diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h
index 7140c208c123..e05dc7649366 100644
--- a/kernel/dma/direct.h
+++ b/kernel/dma/direct.h
@@ -101,15 +101,15 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev,
if (attrs & DMA_ATTR_MMIO) {
dma_addr = phys;
- if (unlikely(!dma_capable(dev, dma_addr, size, false)))
+ if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs)))
goto err_overflow;
} else if (attrs & DMA_ATTR_CC_SHARED) {
dma_addr = phys_to_dma_unencrypted(dev, phys);
- if (unlikely(!dma_capable(dev, dma_addr, size, false)))
+ if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs)))
goto err_overflow;
} else {
dma_addr = phys_to_dma(dev, phys);
- if (unlikely(!dma_capable(dev, dma_addr, size, true)) ||
+ if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) ||
dma_kmalloc_needs_bounce(dev, size, dir)) {
if (is_swiotlb_active(dev) &&
!(attrs & DMA_ATTR_REQUIRE_COHERENT))
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 2bf3981db35d..f4e8b241a1c4 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1678,7 +1678,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
else
dma_addr = phys_to_dma_encrypted(dev, swiotlb_addr);
- if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
+ if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs))) {
__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC,
swiotlb_find_pool(dev, swiotlb_addr));
--
2.43.0
^ permalink raw reply related
* [PATCH v5 08/20] dma-mapping: make dma_pgprot() honor DMA_ATTR_CC_SHARED
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Fold encrypted/decrypted pgprot selection into dma_pgprot() so callers
do not need to adjust the page protection separately.
Update dma_pgprot() to apply pgprot_decrypted() when
DMA_ATTR_CC_SHARED is set and pgprot_encrypted() otherwise Convert
the dma-direct allocation and mmap paths to pass DMA_ATTR_CC_SHARED
instead of open-coding force_dma_unencrypted() handling around
dma_pgprot().
Tested-by: Jiri Pirko <jiri@nvidia.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 8 +++-----
kernel/dma/mapping.c | 16 ++++++++++++----
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 7cf1618a235d..429791b2599a 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -290,9 +290,6 @@ void *dma_direct_alloc(struct device *dev, size_t size,
if (remap) {
pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
- if (force_dma_unencrypted(dev))
- prot = pgprot_decrypted(prot);
-
/* remove any dirty cache lines on the kernel alias */
arch_dma_prep_coherent(page, size);
@@ -614,9 +611,10 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr));
int ret = -ENXIO;
- vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
if (force_dma_unencrypted(dev))
- vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
+ attrs |= DMA_ATTR_CC_SHARED;
+
+ vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
return ret;
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 23ed8eb9233e..44f715f3aa2d 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -543,13 +543,21 @@ EXPORT_SYMBOL(dma_get_sgtable_attrs);
*/
pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs)
{
+ pgprot_t dma_prot;
+
if (dev_is_dma_coherent(dev))
- return prot;
+ dma_prot = prot;
#ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE
- if (attrs & DMA_ATTR_WRITE_COMBINE)
- return pgprot_writecombine(prot);
+ else if (attrs & DMA_ATTR_WRITE_COMBINE)
+ dma_prot = pgprot_writecombine(prot);
#endif
- return pgprot_dmacoherent(prot);
+ else
+ dma_prot = pgprot_dmacoherent(prot);
+
+ if (attrs & DMA_ATTR_CC_SHARED)
+ return pgprot_decrypted(dma_prot);
+ else
+ return pgprot_encrypted(dma_prot);
}
#endif /* CONFIG_MMU */
--
2.43.0
^ permalink raw reply related
* [PATCH v5 07/20] dma: swiotlb: track pool encryption state and honor DMA_ATTR_CC_SHARED
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Teach swiotlb to distinguish between encrypted and decrypted bounce
buffer pools, and make allocation and mapping paths select a pool whose
state matches the requested DMA attributes.
Add a unencrypted flag to io_tlb_mem, initialize it for the default and
restricted pools, and propagate DMA_ATTR_CC_SHARED into swiotlb pool
allocation. Reject swiotlb alloc/map requests when the selected pool does
not match the required encrypted/decrypted state.
Also return DMA addresses with the matching phys_to_dma_{encrypted,
unencrypted} helper so the DMA address encoding stays consistent with the
chosen pool.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
include/linux/dma-direct.h | 10 +++
include/linux/swiotlb.h | 8 +-
kernel/dma/direct.c | 13 +++-
kernel/dma/swiotlb.c | 154 ++++++++++++++++++++++++++++---------
4 files changed, 142 insertions(+), 43 deletions(-)
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index c249912456f9..94fad4e7c11e 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -77,6 +77,10 @@ static inline dma_addr_t dma_range_map_max(const struct bus_dma_region *map)
#ifndef phys_to_dma_unencrypted
#define phys_to_dma_unencrypted phys_to_dma
#endif
+
+#ifndef phys_to_dma_encrypted
+#define phys_to_dma_encrypted phys_to_dma
+#endif
#else
static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
{
@@ -90,6 +94,12 @@ static inline dma_addr_t phys_to_dma_unencrypted(struct device *dev,
{
return dma_addr_unencrypted(__phys_to_dma(dev, paddr));
}
+
+static inline dma_addr_t phys_to_dma_encrypted(struct device *dev,
+ phys_addr_t paddr)
+{
+ return dma_addr_encrypted(__phys_to_dma(dev, paddr));
+}
/*
* If memory encryption is supported, phys_to_dma will set the memory encryption
* bit in the DMA address, and dma_to_phys will clear it.
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 29187cec90d8..4dcbf3931be1 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -81,6 +81,7 @@ struct io_tlb_pool {
struct list_head node;
struct rcu_head rcu;
bool transient;
+ bool unencrypted;
#endif
};
@@ -111,6 +112,7 @@ struct io_tlb_mem {
struct dentry *debugfs;
bool force_bounce;
bool for_alloc;
+ bool unencrypted;
#ifdef CONFIG_SWIOTLB_DYNAMIC
bool can_grow;
u64 phys_limit;
@@ -282,7 +284,8 @@ static inline void swiotlb_sync_single_for_cpu(struct device *dev,
extern void swiotlb_print_info(void);
#ifdef CONFIG_DMA_RESTRICTED_POOL
-struct page *swiotlb_alloc(struct device *dev, size_t size);
+struct page *swiotlb_alloc(struct device *dev, size_t size,
+ unsigned long attrs);
bool swiotlb_free(struct device *dev, struct page *page, size_t size);
void swiotlb_free_from_pool(struct device *dev, phys_addr_t tlb_addr,
size_t size, struct io_tlb_pool *pool);
@@ -292,7 +295,8 @@ static inline bool is_swiotlb_for_alloc(struct device *dev)
return dev->dma_io_tlb_mem->for_alloc;
}
#else
-static inline struct page *swiotlb_alloc(struct device *dev, size_t size)
+static inline struct page *swiotlb_alloc(struct device *dev, size_t size,
+ unsigned long attrs)
{
return NULL;
}
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index dd959716df33..7cf1618a235d 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -96,9 +96,10 @@ static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)
return ret;
}
-static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)
+static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size,
+ unsigned long attrs)
{
- struct page *page = swiotlb_alloc(dev, size);
+ struct page *page = swiotlb_alloc(dev, size, attrs);
if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
swiotlb_free(dev, page, size);
@@ -258,8 +259,12 @@ void *dma_direct_alloc(struct device *dev, size_t size,
gfp, attrs);
if (is_swiotlb_for_alloc(dev)) {
- page = dma_direct_alloc_swiotlb(dev, size);
+ page = dma_direct_alloc_swiotlb(dev, size, attrs);
if (page) {
+ /*
+ * swiotlb allocations comes from pool already marked
+ * decrypted
+ */
mark_mem_decrypt = false;
goto setup_page;
}
@@ -407,7 +412,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
gfp, attrs);
if (is_swiotlb_for_alloc(dev)) {
- page = dma_direct_alloc_swiotlb(dev, size);
+ page = dma_direct_alloc_swiotlb(dev, size, attrs);
if (!page)
return NULL;
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 78ce05857c00..2bf3981db35d 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -259,10 +259,21 @@ void __init swiotlb_update_mem_attributes(void)
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
unsigned long bytes;
+ /*
+ * if platform support memory encryption, swiotlb buffers are
+ * decrypted by default.
+ */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+ io_tlb_default_mem.unencrypted = true;
+ else
+ io_tlb_default_mem.unencrypted = false;
+
if (!mem->nslabs || mem->late_alloc)
return;
bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
- set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
+
+ if (io_tlb_default_mem.unencrypted)
+ set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
}
static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
@@ -505,8 +516,10 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
if (!mem->slots)
goto error_slots;
- set_memory_decrypted((unsigned long)vstart,
- (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+ if (io_tlb_default_mem.unencrypted)
+ set_memory_decrypted((unsigned long)vstart,
+ (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+
swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true,
nareas);
add_mem_pool(&io_tlb_default_mem, mem);
@@ -539,7 +552,9 @@ void __init swiotlb_exit(void)
tbl_size = PAGE_ALIGN(mem->end - mem->start);
slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
- set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+ if (io_tlb_default_mem.unencrypted)
+ set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+
if (mem->late_alloc) {
area_order = get_order(array_size(sizeof(*mem->areas),
mem->nareas));
@@ -563,6 +578,7 @@ void __init swiotlb_exit(void)
* @gfp: GFP flags for the allocation.
* @bytes: Size of the buffer.
* @phys_limit: Maximum allowed physical address of the buffer.
+ * @unencrypted: true to allocate unencrypted memory, false for encrypted memory
*
* Allocate pages from the buddy allocator. If successful, make the allocated
* pages decrypted that they can be used for DMA.
@@ -570,7 +586,8 @@ void __init swiotlb_exit(void)
* Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
* if the allocated physical address was above @phys_limit.
*/
-static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
+static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes,
+ u64 phys_limit, bool unencrypted)
{
unsigned int order = get_order(bytes);
struct page *page;
@@ -588,13 +605,13 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
}
vaddr = phys_to_virt(paddr);
- if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
+ if (unencrypted && set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
goto error;
return page;
error:
/* Intentional leak if pages cannot be encrypted again. */
- if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
+ if (unencrypted && !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
__free_pages(page, order);
return NULL;
}
@@ -604,30 +621,26 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
* @dev: Device for which a memory pool is allocated.
* @bytes: Size of the buffer.
* @phys_limit: Maximum allowed physical address of the buffer.
+ * @attrs: DMA attributes for the allocation.
* @gfp: GFP flags for the allocation.
*
* Return: Allocated pages, or %NULL on allocation failure.
*/
static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
- u64 phys_limit, gfp_t gfp)
+ u64 phys_limit, unsigned long attrs, gfp_t gfp)
{
struct page *page;
- unsigned long attrs = 0;
/*
* Allocate from the atomic pools if memory is encrypted and
* the allocation is atomic, because decrypting may block.
*/
- if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
+ if (!gfpflags_allow_blocking(gfp) && (attrs & DMA_ATTR_CC_SHARED)) {
void *vaddr;
if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
return NULL;
- /* swiotlb considered decrypted by default */
- if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
- attrs = DMA_ATTR_CC_SHARED;
-
return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
attrs, dma_coherent_ok);
}
@@ -638,7 +651,8 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
else if (phys_limit <= DMA_BIT_MASK(32))
gfp |= __GFP_DMA32;
- while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit))) {
+ while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit,
+ !!(attrs & DMA_ATTR_CC_SHARED)))) {
if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
phys_limit < DMA_BIT_MASK(64) &&
!(gfp & (__GFP_DMA32 | __GFP_DMA)))
@@ -657,15 +671,18 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
* swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
* @vaddr: Virtual address of the buffer.
* @bytes: Size of the buffer.
+ * @unencrypted: true if @vaddr was allocated decrypted and must be
+ * re-encrypted before being freed
*/
-static void swiotlb_free_tlb(void *vaddr, size_t bytes)
+static void swiotlb_free_tlb(void *vaddr, size_t bytes, bool unencrypted)
{
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
dma_free_from_pool(NULL, vaddr, bytes))
return;
/* Intentional leak if pages cannot be encrypted again. */
- if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
+ if (!unencrypted ||
+ !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
__free_pages(virt_to_page(vaddr), get_order(bytes));
}
@@ -676,6 +693,7 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes)
* @nslabs: Desired (maximum) number of slabs.
* @nareas: Number of areas.
* @phys_limit: Maximum DMA buffer physical address.
+ * @attrs: DMA attributes for the allocation.
* @gfp: GFP flags for the allocations.
*
* Allocate and initialize a new IO TLB memory pool. The actual number of
@@ -686,7 +704,8 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes)
*/
static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
unsigned long minslabs, unsigned long nslabs,
- unsigned int nareas, u64 phys_limit, gfp_t gfp)
+ unsigned int nareas, u64 phys_limit,
+ unsigned long attrs, gfp_t gfp)
{
struct io_tlb_pool *pool;
unsigned int slot_order;
@@ -704,9 +723,10 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
if (!pool)
goto error;
pool->areas = (void *)pool + sizeof(*pool);
+ pool->unencrypted = !!(attrs & DMA_ATTR_CC_SHARED);
tlb_size = nslabs << IO_TLB_SHIFT;
- while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) {
+ while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, attrs, gfp))) {
if (nslabs <= minslabs)
goto error_tlb;
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
@@ -724,7 +744,8 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
return pool;
error_slots:
- swiotlb_free_tlb(page_address(tlb), tlb_size);
+ swiotlb_free_tlb(page_address(tlb), tlb_size,
+ !!(attrs & DMA_ATTR_CC_SHARED));
error_tlb:
kfree(pool);
error:
@@ -742,7 +763,9 @@ static void swiotlb_dyn_alloc(struct work_struct *work)
struct io_tlb_pool *pool;
pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
- default_nareas, mem->phys_limit, GFP_KERNEL);
+ default_nareas, mem->phys_limit,
+ mem->unencrypted ? DMA_ATTR_CC_SHARED : 0,
+ GFP_KERNEL);
if (!pool) {
pr_warn_ratelimited("Failed to allocate new pool");
return;
@@ -762,7 +785,7 @@ static void swiotlb_dyn_free(struct rcu_head *rcu)
size_t tlb_size = pool->end - pool->start;
free_pages((unsigned long)pool->slots, get_order(slots_size));
- swiotlb_free_tlb(pool->vaddr, tlb_size);
+ swiotlb_free_tlb(pool->vaddr, tlb_size, pool->unencrypted);
kfree(pool);
}
@@ -1037,13 +1060,11 @@ static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
* Return: Index of the first allocated slot, or -1 on error.
*/
static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
- int area_index, phys_addr_t orig_addr, size_t alloc_size,
- unsigned int alloc_align_mask)
+ int area_index, phys_addr_t orig_addr, dma_addr_t tbl_dma_addr,
+ size_t alloc_size, unsigned int alloc_align_mask)
{
struct io_tlb_area *area = pool->areas + area_index;
unsigned long boundary_mask = dma_get_seg_boundary(dev);
- dma_addr_t tbl_dma_addr =
- phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
unsigned long max_slots = get_max_slots(boundary_mask);
unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
unsigned int nslots = nr_slots(alloc_size), stride;
@@ -1056,6 +1077,8 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
BUG_ON(!nslots);
BUG_ON(area_index >= pool->nareas);
+ tbl_dma_addr &= boundary_mask;
+
/*
* Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
* page-aligned in the absence of any other alignment requirements.
@@ -1167,6 +1190,7 @@ static int swiotlb_search_area(struct device *dev, int start_cpu,
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
+ dma_addr_t tbl_dma_addr;
int area_index;
int index = -1;
@@ -1175,9 +1199,15 @@ static int swiotlb_search_area(struct device *dev, int start_cpu,
if (cpu_offset >= pool->nareas)
continue;
area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
+
+ if (mem->unencrypted)
+ tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
+ else
+ tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
+
index = swiotlb_search_pool_area(dev, pool, area_index,
- orig_addr, alloc_size,
- alloc_align_mask);
+ orig_addr, tbl_dma_addr,
+ alloc_size, alloc_align_mask);
if (index >= 0) {
*retpool = pool;
break;
@@ -1207,6 +1237,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
+ dma_addr_t tbl_dma_addr;
unsigned long nslabs;
unsigned long flags;
u64 phys_limit;
@@ -1232,11 +1263,17 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
nslabs = nr_slots(alloc_size);
phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
+ mem->unencrypted ? DMA_ATTR_CC_SHARED : 0,
GFP_NOWAIT);
if (!pool)
return -1;
- index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,
+ if (mem->unencrypted)
+ tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
+ else
+ tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
+
+ index = swiotlb_search_pool_area(dev, pool, 0, orig_addr, tbl_dma_addr,
alloc_size, alloc_align_mask);
if (index < 0) {
swiotlb_dyn_free(&pool->rcu);
@@ -1281,15 +1318,23 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
size_t alloc_size, unsigned int alloc_align_mask,
struct io_tlb_pool **retpool)
{
+ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
+ dma_addr_t tbl_dma_addr;
int start, i;
int index;
- *retpool = pool = &dev->dma_io_tlb_mem->defpool;
+ *retpool = pool = &mem->defpool;
+ if (mem->unencrypted)
+ tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
+ else
+ tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
+
i = start = raw_smp_processor_id() & (pool->nareas - 1);
do {
index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
- alloc_size, alloc_align_mask);
+ tbl_dma_addr, alloc_size,
+ alloc_align_mask);
if (index >= 0)
return index;
if (++i >= pool->nareas)
@@ -1372,9 +1417,19 @@ static unsigned long mem_used(struct io_tlb_mem *mem)
* any pre- or post-padding for alignment
* @alloc_align_mask: Required start and end alignment of the allocated buffer
* @dir: DMA direction
- * @attrs: Optional DMA attributes for the map operation
+ * @attrs: Optional DMA attributes for the map operation, updated
+ * to match the selected SWIOTLB pool
*
* Find and allocate a suitable sequence of IO TLB slots for the request.
+ * The device's SWIOTLB pool must match the device's current DMA encryption
+ * requirements. If the device requires decrypted DMA, bouncing is done through
+ * an unencrypted pool and the mapping is marked shared. If the device can DMA
+ * to encrypted memory, bouncing is done through an encrypted pool even when the
+ * original DMA address was unencrypted. Enabling encrypted DMA for a device is
+ * therefore expected to update its default io_tlb_mem to an encrypted pool, so
+ * later bounce mappings for both encrypted and decrypted original memory use
+ * that encrypted pool.
+ *
* The allocated space starts at an alignment specified by alloc_align_mask,
* and the size of the allocated space is rounded up so that the total amount
* of allocated space is a multiple of (alloc_align_mask + 1). If
@@ -1411,6 +1466,16 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
+ /* swiotlb pool is incorrect for this device */
+ if (unlikely(mem->unencrypted != force_dma_unencrypted(dev)))
+ return (phys_addr_t)DMA_MAPPING_ERROR;
+
+ /* Force attrs to match the kind of memory in the pool */
+ if (mem->unencrypted)
+ *attrs |= DMA_ATTR_CC_SHARED;
+ else
+ *attrs &= ~DMA_ATTR_CC_SHARED;
+
/*
* The default swiotlb memory pool is allocated with PAGE_SIZE
* alignment. If a mapping is requested with larger alignment,
@@ -1608,8 +1673,11 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
- /* Ensure that the address returned is DMA'ble */
- dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
+ if (attrs & DMA_ATTR_CC_SHARED)
+ dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
+ else
+ dma_addr = phys_to_dma_encrypted(dev, swiotlb_addr);
+
if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC,
@@ -1773,7 +1841,7 @@ static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
#ifdef CONFIG_DMA_RESTRICTED_POOL
-struct page *swiotlb_alloc(struct device *dev, size_t size)
+struct page *swiotlb_alloc(struct device *dev, size_t size, unsigned long attrs)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
@@ -1784,6 +1852,9 @@ struct page *swiotlb_alloc(struct device *dev, size_t size)
if (!mem)
return NULL;
+ if (mem->unencrypted != !!(attrs & DMA_ATTR_CC_SHARED))
+ return NULL;
+
align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
index = swiotlb_find_slots(dev, 0, size, align, &pool);
if (index == -1)
@@ -1859,9 +1930,18 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
kfree(mem);
return -ENOMEM;
}
+ /*
+ * if platform supports memory encryption,
+ * restricted mem pool is decrypted by default
+ */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
+ mem->unencrypted = true;
+ set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
+ rmem->size >> PAGE_SHIFT);
+ } else {
+ mem->unencrypted = false;
+ }
- set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
- rmem->size >> PAGE_SHIFT);
swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs,
false, nareas);
mem->force_bounce = true;
--
2.43.0
^ permalink raw reply related
* [PATCH v5 06/20] dma: swiotlb: pass mapping attributes by reference
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Change swiotlb_tbl_map_single() to take the DMA mapping attributes by
reference and update the direct callers accordingly.
This is a preparatory change for a follow-up patch which updates the
attributes based on the selected swiotlb pool. Keeping the signature change
separate makes the follow-up patch easier to review.
No functional change in this patch.
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
drivers/iommu/dma-iommu.c | 2 +-
drivers/xen/swiotlb-xen.c | 2 +-
include/linux/swiotlb.h | 2 +-
kernel/dma/swiotlb.c | 6 +++---
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index c2595bee3d41..725c7adb0a8d 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1180,7 +1180,7 @@ static phys_addr_t iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys,
trace_swiotlb_bounced(dev, phys, size);
phys = swiotlb_tbl_map_single(dev, phys, size, iova_mask(iovad), dir,
- attrs);
+ &attrs);
/*
* Untrusted devices should not see padding areas with random leftover
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 2cbf2b588f5b..8c4abe65cd49 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -243,7 +243,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
*/
trace_swiotlb_bounced(dev, dev_addr, size);
- map = swiotlb_tbl_map_single(dev, phys, size, 0, dir, attrs);
+ map = swiotlb_tbl_map_single(dev, phys, size, 0, dir, &attrs);
if (map == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 133bb8ca9032..29187cec90d8 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -238,7 +238,7 @@ static inline phys_addr_t default_swiotlb_limit(void)
phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
size_t mapping_size, unsigned int alloc_aligned_mask,
- enum dma_data_direction dir, unsigned long attrs);
+ enum dma_data_direction dir, unsigned long *attrs);
dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
size_t size, enum dma_data_direction dir, unsigned long attrs);
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index be4d418d92ac..78ce05857c00 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1391,7 +1391,7 @@ static unsigned long mem_used(struct io_tlb_mem *mem)
*/
phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
size_t mapping_size, unsigned int alloc_align_mask,
- enum dma_data_direction dir, unsigned long attrs)
+ enum dma_data_direction dir, unsigned long *attrs)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
unsigned int offset;
@@ -1425,7 +1425,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
size = ALIGN(mapping_size + offset, alloc_align_mask + 1);
index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool);
if (index == -1) {
- if (!(attrs & DMA_ATTR_NO_WARN))
+ if (!(*attrs & DMA_ATTR_NO_WARN))
dev_warn_ratelimited(dev,
"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
size, mem->nslabs, mem_used(mem));
@@ -1604,7 +1604,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
- swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);
+ swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, &attrs);
if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
--
2.43.0
^ permalink raw reply related
* [PATCH v5 05/20] dma-pool: track decrypted atomic pools and select them via attrs
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:28 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Teach the atomic DMA pool code to distinguish between encrypted and
unencrypted pools, and make pool allocation select the matching pool based
on DMA attributes.
Introduce a dma_gen_pool wrapper that records whether a pool is
unencrypted, initialize that state when the atomic pools are created, and
use it when expanding and resizing the pools. Update dma_alloc_from_pool()
to take attrs and skip pools whose encrypted state does not match
DMA_ATTR_CC_SHARED. Update dma_free_from_pool() accordingly.
Also pass DMA_ATTR_CC_SHARED from the swiotlb atomic allocation path so
decrypted swiotlb allocations are taken from the correct atomic pool.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Reviewed-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
drivers/iommu/dma-iommu.c | 2 +-
include/linux/dma-map-ops.h | 2 +-
kernel/dma/direct.c | 11 ++-
kernel/dma/pool.c | 167 +++++++++++++++++++++++-------------
kernel/dma/swiotlb.c | 7 +-
5 files changed, 123 insertions(+), 66 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 54d96e847f16..c2595bee3d41 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1673,7 +1673,7 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
!gfpflags_allow_blocking(gfp) && !coherent)
page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
- gfp, NULL);
+ gfp, attrs, NULL);
else
cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
if (!cpu_addr)
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index 6a1832a73cad..696b2c3a2305 100644
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -212,7 +212,7 @@ void *dma_common_pages_remap(struct page **pages, size_t size, pgprot_t prot,
void dma_common_free_remap(void *cpu_addr, size_t size);
struct page *dma_alloc_from_pool(struct device *dev, size_t size,
- void **cpu_addr, gfp_t flags,
+ void **cpu_addr, gfp_t flags, unsigned long attrs,
bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
bool dma_free_from_pool(struct device *dev, void *start, size_t size);
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index a224b1bed6f9..dd959716df33 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -154,7 +154,7 @@ static bool dma_direct_use_pool(struct device *dev, gfp_t gfp)
}
static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
- dma_addr_t *dma_handle, gfp_t gfp)
+ dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
struct page *page;
u64 phys_limit;
@@ -164,7 +164,8 @@ static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,
return NULL;
gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
- page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);
+ page = dma_alloc_from_pool(dev, size, &ret, gfp, attrs,
+ dma_coherent_ok);
if (!page)
return NULL;
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
@@ -253,7 +254,8 @@ void *dma_direct_alloc(struct device *dev, size_t size,
*/
if ((remap || (attrs & DMA_ATTR_CC_SHARED)) &&
dma_direct_use_pool(dev, gfp))
- return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+ return dma_direct_alloc_from_pool(dev, size, dma_handle,
+ gfp, attrs);
if (is_swiotlb_for_alloc(dev)) {
page = dma_direct_alloc_swiotlb(dev, size);
@@ -401,7 +403,8 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
attrs |= DMA_ATTR_CC_SHARED;
if ((attrs & DMA_ATTR_CC_SHARED) && dma_direct_use_pool(dev, gfp))
- return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+ return dma_direct_alloc_from_pool(dev, size, dma_handle,
+ gfp, attrs);
if (is_swiotlb_for_alloc(dev)) {
page = dma_direct_alloc_swiotlb(dev, size);
diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c
index 2b2fbb709242..be78474a6c49 100644
--- a/kernel/dma/pool.c
+++ b/kernel/dma/pool.c
@@ -12,12 +12,18 @@
#include <linux/set_memory.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
+#include <linux/cc_platform.h>
-static struct gen_pool *atomic_pool_dma __ro_after_init;
+struct dma_gen_pool {
+ bool unencrypted;
+ struct gen_pool *pool;
+};
+
+static struct dma_gen_pool atomic_pool_dma __ro_after_init;
static unsigned long pool_size_dma;
-static struct gen_pool *atomic_pool_dma32 __ro_after_init;
+static struct dma_gen_pool atomic_pool_dma32 __ro_after_init;
static unsigned long pool_size_dma32;
-static struct gen_pool *atomic_pool_kernel __ro_after_init;
+static struct dma_gen_pool atomic_pool_kernel __ro_after_init;
static unsigned long pool_size_kernel;
/* Size can be defined by the coherent_pool command line */
@@ -76,11 +82,12 @@ static bool cma_in_zone(gfp_t gfp)
return true;
}
-static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
+static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_size,
gfp_t gfp)
{
unsigned int order;
struct page *page = NULL;
+ bool leak_pages = false;
void *addr;
int ret = -ENOMEM;
@@ -113,12 +120,17 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
* Memory in the atomic DMA pools must be unencrypted, the pools do not
* shrink so no re-encryption occurs in dma_direct_free().
*/
- ret = set_memory_decrypted((unsigned long)page_to_virt(page),
- 1 << order);
- if (ret)
- goto remove_mapping;
- ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),
- pool_size, NUMA_NO_NODE);
+ if (dma_pool->unencrypted) {
+ ret = set_memory_decrypted((unsigned long)page_to_virt(page),
+ 1 << order);
+ if (ret) {
+ leak_pages = true;
+ goto remove_mapping;
+ }
+ }
+
+ ret = gen_pool_add_virt(dma_pool->pool, (unsigned long)addr,
+ page_to_phys(page), pool_size, NUMA_NO_NODE);
if (ret)
goto encrypt_mapping;
@@ -126,62 +138,67 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
return 0;
encrypt_mapping:
- ret = set_memory_encrypted((unsigned long)page_to_virt(page),
- 1 << order);
- if (WARN_ON_ONCE(ret)) {
- /* Decrypt succeeded but encrypt failed, purposely leak */
- goto out;
- }
+ if (dma_pool->unencrypted &&
+ set_memory_encrypted((unsigned long)page_to_virt(page), 1 << order))
+ leak_pages = true;
+
remove_mapping:
#ifdef CONFIG_DMA_DIRECT_REMAP
dma_common_free_remap(addr, pool_size);
free_page:
- __free_pages(page, order);
+ if (!leak_pages)
+ __free_pages(page, order);
#endif
out:
return ret;
}
-static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp)
+static void atomic_pool_resize(struct dma_gen_pool *dma_pool, gfp_t gfp)
{
- if (pool && gen_pool_avail(pool) < atomic_pool_size)
- atomic_pool_expand(pool, gen_pool_size(pool), gfp);
+ if (dma_pool->pool && gen_pool_avail(dma_pool->pool) < atomic_pool_size)
+ atomic_pool_expand(dma_pool, gen_pool_size(dma_pool->pool), gfp);
}
static void atomic_pool_work_fn(struct work_struct *work)
{
if (IS_ENABLED(CONFIG_ZONE_DMA))
- atomic_pool_resize(atomic_pool_dma,
+ atomic_pool_resize(&atomic_pool_dma,
GFP_KERNEL | GFP_DMA);
if (IS_ENABLED(CONFIG_ZONE_DMA32))
- atomic_pool_resize(atomic_pool_dma32,
+ atomic_pool_resize(&atomic_pool_dma32,
GFP_KERNEL | GFP_DMA32);
- atomic_pool_resize(atomic_pool_kernel, GFP_KERNEL);
+ atomic_pool_resize(&atomic_pool_kernel, GFP_KERNEL);
}
-static __init struct gen_pool *__dma_atomic_pool_init(size_t pool_size,
- gfp_t gfp)
+static __init struct dma_gen_pool *__dma_atomic_pool_init(struct dma_gen_pool *dma_pool,
+ size_t pool_size, gfp_t gfp)
{
- struct gen_pool *pool;
int ret;
- pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
- if (!pool)
+ dma_pool->pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
+ if (!dma_pool->pool)
return NULL;
- gen_pool_set_algo(pool, gen_pool_first_fit_order_align, NULL);
+ gen_pool_set_algo(dma_pool->pool, gen_pool_first_fit_order_align, NULL);
+
+ /* if platform is using memory encryption atomic pools are by default decrypted. */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+ dma_pool->unencrypted = true;
+ else
+ dma_pool->unencrypted = false;
- ret = atomic_pool_expand(pool, pool_size, gfp);
+ ret = atomic_pool_expand(dma_pool, pool_size, gfp);
if (ret) {
- gen_pool_destroy(pool);
+ gen_pool_destroy(dma_pool->pool);
+ dma_pool->pool = NULL;
pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
pool_size >> 10, &gfp);
return NULL;
}
pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",
- gen_pool_size(pool) >> 10, &gfp);
- return pool;
+ gen_pool_size(dma_pool->pool) >> 10, &gfp);
+ return dma_pool;
}
#ifdef CONFIG_ZONE_DMA32
@@ -207,21 +224,22 @@ static int __init dma_atomic_pool_init(void)
/* All memory might be in the DMA zone(s) to begin with */
if (has_managed_zone(ZONE_NORMAL)) {
- atomic_pool_kernel = __dma_atomic_pool_init(atomic_pool_size,
- GFP_KERNEL);
- if (!atomic_pool_kernel)
+ __dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size, GFP_KERNEL);
+ if (!atomic_pool_kernel.pool)
ret = -ENOMEM;
}
+
if (has_managed_dma()) {
- atomic_pool_dma = __dma_atomic_pool_init(atomic_pool_size,
- GFP_KERNEL | GFP_DMA);
- if (!atomic_pool_dma)
+ __dma_atomic_pool_init(&atomic_pool_dma, atomic_pool_size,
+ GFP_KERNEL | GFP_DMA);
+ if (!atomic_pool_dma.pool)
ret = -ENOMEM;
}
+
if (has_managed_dma32) {
- atomic_pool_dma32 = __dma_atomic_pool_init(atomic_pool_size,
- GFP_KERNEL | GFP_DMA32);
- if (!atomic_pool_dma32)
+ __dma_atomic_pool_init(&atomic_pool_dma32, atomic_pool_size,
+ GFP_KERNEL | GFP_DMA32);
+ if (!atomic_pool_dma32.pool)
ret = -ENOMEM;
}
@@ -230,19 +248,44 @@ static int __init dma_atomic_pool_init(void)
}
postcore_initcall(dma_atomic_pool_init);
-static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp)
+static inline struct dma_gen_pool *__dma_guess_pool(struct dma_gen_pool *first,
+ struct dma_gen_pool *second, struct dma_gen_pool *third)
{
- if (prev == NULL) {
+ if (first->pool)
+ return first;
+ if (second && second->pool)
+ return second;
+ if (third && third->pool)
+ return third;
+ return NULL;
+}
+
+static inline struct dma_gen_pool *dma_guess_pool(struct dma_gen_pool *prev,
+ gfp_t gfp)
+{
+ if (!prev) {
if (gfp & GFP_DMA)
- return atomic_pool_dma ?: atomic_pool_dma32 ?: atomic_pool_kernel;
+ return __dma_guess_pool(&atomic_pool_dma,
+ &atomic_pool_dma32,
+ &atomic_pool_kernel);
+
if (gfp & GFP_DMA32)
- return atomic_pool_dma32 ?: atomic_pool_dma ?: atomic_pool_kernel;
- return atomic_pool_kernel ?: atomic_pool_dma32 ?: atomic_pool_dma;
+ return __dma_guess_pool(&atomic_pool_dma32,
+ &atomic_pool_dma,
+ &atomic_pool_kernel);
+
+ return __dma_guess_pool(&atomic_pool_kernel,
+ &atomic_pool_dma32,
+ &atomic_pool_dma);
}
- if (prev == atomic_pool_kernel)
- return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma;
- if (prev == atomic_pool_dma32)
- return atomic_pool_dma;
+
+ if (prev == &atomic_pool_kernel)
+ return __dma_guess_pool(&atomic_pool_dma32,
+ &atomic_pool_dma, NULL);
+
+ if (prev == &atomic_pool_dma32)
+ return __dma_guess_pool(&atomic_pool_dma, NULL, NULL);
+
return NULL;
}
@@ -272,16 +315,20 @@ static struct page *__dma_alloc_from_pool(struct device *dev, size_t size,
}
struct page *dma_alloc_from_pool(struct device *dev, size_t size,
- void **cpu_addr, gfp_t gfp,
+ void **cpu_addr, gfp_t gfp, unsigned long attrs,
bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t))
{
- struct gen_pool *pool = NULL;
+ struct dma_gen_pool *dma_pool = NULL;
struct page *page;
bool pool_found = false;
- while ((pool = dma_guess_pool(pool, gfp))) {
+ while ((dma_pool = dma_guess_pool(dma_pool, gfp))) {
+
+ if (dma_pool->unencrypted != !!(attrs & DMA_ATTR_CC_SHARED))
+ continue;
+
pool_found = true;
- page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,
+ page = __dma_alloc_from_pool(dev, size, dma_pool->pool, cpu_addr,
phys_addr_ok);
if (page)
return page;
@@ -296,12 +343,14 @@ struct page *dma_alloc_from_pool(struct device *dev, size_t size,
bool dma_free_from_pool(struct device *dev, void *start, size_t size)
{
- struct gen_pool *pool = NULL;
+ struct dma_gen_pool *dma_pool = NULL;
+
+ while ((dma_pool = dma_guess_pool(dma_pool, 0))) {
- while ((pool = dma_guess_pool(pool, 0))) {
- if (!gen_pool_has_addr(pool, (unsigned long)start, size))
+ if (!gen_pool_has_addr(dma_pool->pool, (unsigned long)start, size))
continue;
- gen_pool_free(pool, (unsigned long)start, size);
+
+ gen_pool_free(dma_pool->pool, (unsigned long)start, size);
return true;
}
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index ac03a6856c2e..be4d418d92ac 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -612,6 +612,7 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
u64 phys_limit, gfp_t gfp)
{
struct page *page;
+ unsigned long attrs = 0;
/*
* Allocate from the atomic pools if memory is encrypted and
@@ -623,8 +624,12 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
return NULL;
+ /* swiotlb considered decrypted by default */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+ attrs = DMA_ATTR_CC_SHARED;
+
return dma_alloc_from_pool(dev, bytes, &vaddr, gfp,
- dma_coherent_ok);
+ attrs, dma_coherent_ok);
}
gfp &= ~GFP_ZONEMASK;
--
2.43.0
^ permalink raw reply related
* [PATCH v5 04/20] dma-direct: use DMA_ATTR_CC_SHARED in alloc/free paths
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:27 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Propagate force_dma_unencrypted() into DMA_ATTR_CC_SHARED in the
dma-direct allocation path and use the attribute to drive the related
decisions.
This updates dma_direct_alloc(), dma_direct_free(), and
dma_direct_alloc_pages() to fold the forced unencrypted case into attrs.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 53 +++++++++++++++++++++++++++++++++++++--------
1 file changed, 44 insertions(+), 9 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index fe8e83a36058..a224b1bed6f9 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -193,16 +193,31 @@ void *dma_direct_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
bool remap = false, set_uncached = false;
- bool mark_mem_decrypt = true;
+ bool mark_mem_decrypt = false;
struct page *page;
void *ret;
+ /*
+ * DMA_ATTR_CC_SHARED is not a caller-visible dma_alloc_*()
+ * attribute. The direct allocator uses it internally after it has
+ * decided that the backing pages must be shared/decrypted, so the
+ * rest of the allocation path can consistently select DMA addresses,
+ * choose compatible pools and restore encryption on free.
+ */
+ if (attrs & DMA_ATTR_CC_SHARED)
+ return NULL;
+
+ if (force_dma_unencrypted(dev)) {
+ attrs |= DMA_ATTR_CC_SHARED;
+ mark_mem_decrypt = true;
+ }
+
size = PAGE_ALIGN(size);
if (attrs & DMA_ATTR_NO_WARN)
gfp |= __GFP_NOWARN;
- if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
- !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev))
+ if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_CC_SHARED)) ==
+ DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev))
return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
if (!dev_is_dma_coherent(dev)) {
@@ -236,7 +251,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
* Remapping or decrypting memory may block, allocate the memory from
* the atomic pools instead if we aren't allowed block.
*/
- if ((remap || force_dma_unencrypted(dev)) &&
+ if ((remap || (attrs & DMA_ATTR_CC_SHARED)) &&
dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
@@ -312,12 +327,24 @@ void dma_direct_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
{
phys_addr_t phys;
- bool mark_mem_encrypted = true;
+ bool mark_mem_encrypted = false;
struct io_tlb_pool *swiotlb_pool;
unsigned int page_order = get_order(size);
- if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
- !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) {
+ /* see dma_direct_alloc() for details */
+ WARN_ON(attrs & DMA_ATTR_CC_SHARED);
+
+ /*
+ * if the device had requested for an unencrypted buffer,
+ * convert it to encrypted on free
+ */
+ if (force_dma_unencrypted(dev)) {
+ attrs |= DMA_ATTR_CC_SHARED;
+ mark_mem_encrypted = true;
+ }
+
+ if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_CC_SHARED)) ==
+ DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) {
/* cpu_addr is a struct page cookie, not a kernel address */
dma_free_contiguous(dev, cpu_addr, size);
return;
@@ -366,10 +393,14 @@ void dma_direct_free(struct device *dev, size_t size,
struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
{
+ unsigned long attrs = 0;
struct page *page;
void *ret;
- if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
+ if (force_dma_unencrypted(dev))
+ attrs |= DMA_ATTR_CC_SHARED;
+
+ if ((attrs & DMA_ATTR_CC_SHARED) && dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
if (is_swiotlb_for_alloc(dev)) {
@@ -403,7 +434,11 @@ void dma_direct_free_pages(struct device *dev, size_t size,
phys_addr_t phys;
void *vaddr = page_address(page);
struct io_tlb_pool *swiotlb_pool;
- bool mark_mem_encrypted = true;
+ /*
+ * if the device had requested for an unencrypted buffer,
+ * convert it to encrypted on free
+ */
+ bool mark_mem_encrypted = force_dma_unencrypted(dev);
/* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
--
2.43.0
^ permalink raw reply related
* [PATCH v5 03/20] dma-direct: swiotlb: handle swiotlb alloc/free outside __dma_direct_alloc_pages
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:27 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Move swiotlb allocation out of __dma_direct_alloc_pages() and handle it in
dma_direct_alloc() / dma_direct_alloc_pages().
This is needed for follow-up changes that simplify the handling of
memory encryption/decryption based on the DMA attribute flags.
swiotlb backing pages are already mapped decrypted by
swiotlb_update_mem_attributes() and rmem_swiotlb_device_init(), so
dma-direct should not call dma_set_decrypted() on allocation nor
dma_set_encrypted() on free for swiotlb-backed memory.
Update alloc/free paths to detect swiotlb-backed pages and skip
encrypt/decrypt transitions for those paths. Keep the existing highmem
rejection in dma_direct_alloc_pages() for swiotlb allocations.
Only for "restricted-dma-pool", we currently set `for_alloc = true`, while
rmem_swiotlb_device_init() decrypts the whole pool up front. This pool is
typically used together with "shared-dma-pool", where the shared region is
accessed after remap/ioremap and the returned address is suitable for
decrypted memory access. So existing code paths remain valid.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
include/linux/swiotlb.h | 6 ++++
kernel/dma/direct.c | 71 ++++++++++++++++++++++++++++++-----------
kernel/dma/swiotlb.c | 6 ++++
3 files changed, 65 insertions(+), 18 deletions(-)
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 3dae0f592063..133bb8ca9032 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -284,6 +284,8 @@ extern void swiotlb_print_info(void);
#ifdef CONFIG_DMA_RESTRICTED_POOL
struct page *swiotlb_alloc(struct device *dev, size_t size);
bool swiotlb_free(struct device *dev, struct page *page, size_t size);
+void swiotlb_free_from_pool(struct device *dev, phys_addr_t tlb_addr,
+ size_t size, struct io_tlb_pool *pool);
static inline bool is_swiotlb_for_alloc(struct device *dev)
{
@@ -299,6 +301,10 @@ static inline bool swiotlb_free(struct device *dev, struct page *page,
{
return false;
}
+static inline void swiotlb_free_from_pool(struct device *dev, phys_addr_t tlb_addr,
+ size_t size, struct io_tlb_pool *pool)
+{
+}
static inline bool is_swiotlb_for_alloc(struct device *dev)
{
return false;
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index ec887f443741..fe8e83a36058 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -96,14 +96,6 @@ static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)
return ret;
}
-static void __dma_direct_free_pages(struct device *dev, struct page *page,
- size_t size)
-{
- if (swiotlb_free(dev, page, size))
- return;
- dma_free_contiguous(dev, page, size);
-}
-
static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)
{
struct page *page = swiotlb_alloc(dev, size);
@@ -125,9 +117,6 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
WARN_ON_ONCE(!PAGE_ALIGNED(size));
- if (is_swiotlb_for_alloc(dev))
- return dma_direct_alloc_swiotlb(dev, size);
-
gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
page = dma_alloc_contiguous(dev, size, gfp);
if (page) {
@@ -204,6 +193,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
bool remap = false, set_uncached = false;
+ bool mark_mem_decrypt = true;
struct page *page;
void *ret;
@@ -250,11 +240,21 @@ void *dma_direct_alloc(struct device *dev, size_t size,
dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+ if (is_swiotlb_for_alloc(dev)) {
+ page = dma_direct_alloc_swiotlb(dev, size);
+ if (page) {
+ mark_mem_decrypt = false;
+ goto setup_page;
+ }
+ return NULL;
+ }
+
/* we always manually zero the memory once we are done */
page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);
if (!page)
return NULL;
+setup_page:
/*
* dma_alloc_contiguous can return highmem pages depending on a
* combination the cma= arguments and per-arch setup. These need to be
@@ -281,7 +281,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
goto out_free_pages;
} else {
ret = page_address(page);
- if (dma_set_decrypted(dev, ret, size))
+ if (mark_mem_decrypt && dma_set_decrypted(dev, ret, size))
goto out_leak_pages;
}
@@ -298,10 +298,11 @@ void *dma_direct_alloc(struct device *dev, size_t size,
return ret;
out_encrypt_pages:
- if (dma_set_encrypted(dev, page_address(page), size))
+ if (mark_mem_decrypt && dma_set_encrypted(dev, page_address(page), size))
return NULL;
out_free_pages:
- __dma_direct_free_pages(dev, page, size);
+ if (!swiotlb_free(dev, page, size))
+ dma_free_contiguous(dev, page, size);
return NULL;
out_leak_pages:
return NULL;
@@ -310,6 +311,9 @@ void *dma_direct_alloc(struct device *dev, size_t size,
void dma_direct_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
{
+ phys_addr_t phys;
+ bool mark_mem_encrypted = true;
+ struct io_tlb_pool *swiotlb_pool;
unsigned int page_order = get_order(size);
if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
@@ -338,16 +342,25 @@ void dma_direct_free(struct device *dev, size_t size,
dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))
return;
+ phys = dma_to_phys(dev, dma_addr);
+ swiotlb_pool = swiotlb_find_pool(dev, phys);
+ if (swiotlb_pool)
+ /* Swiotlb doesn't need a page attribute update on free */
+ mark_mem_encrypted = false;
+
if (is_vmalloc_addr(cpu_addr)) {
vunmap(cpu_addr);
} else {
if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
arch_dma_clear_uncached(cpu_addr, size);
- if (dma_set_encrypted(dev, cpu_addr, size))
+ if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size))
return;
}
- __dma_direct_free_pages(dev, dma_direct_to_page(dev, dma_addr), size);
+ if (swiotlb_pool)
+ swiotlb_free_from_pool(dev, phys, size, swiotlb_pool);
+ else
+ dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size);
}
struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
@@ -359,6 +372,15 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);
+ if (is_swiotlb_for_alloc(dev)) {
+ page = dma_direct_alloc_swiotlb(dev, size);
+ if (!page)
+ return NULL;
+
+ ret = page_address(page);
+ goto setup_page;
+ }
+
page = __dma_direct_alloc_pages(dev, size, gfp, false);
if (!page)
return NULL;
@@ -366,6 +388,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
ret = page_address(page);
if (dma_set_decrypted(dev, ret, size))
goto out_leak_pages;
+setup_page:
memset(ret, 0, size);
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
return page;
@@ -377,16 +400,28 @@ void dma_direct_free_pages(struct device *dev, size_t size,
struct page *page, dma_addr_t dma_addr,
enum dma_data_direction dir)
{
+ phys_addr_t phys;
void *vaddr = page_address(page);
+ struct io_tlb_pool *swiotlb_pool;
+ bool mark_mem_encrypted = true;
/* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
dma_free_from_pool(dev, vaddr, size))
return;
- if (dma_set_encrypted(dev, vaddr, size))
+ phys = page_to_phys(page);
+ swiotlb_pool = swiotlb_find_pool(dev, phys);
+ if (swiotlb_pool)
+ mark_mem_encrypted = false;
+
+ if (mark_mem_encrypted && dma_set_encrypted(dev, vaddr, size))
return;
- __dma_direct_free_pages(dev, page, size);
+
+ if (swiotlb_pool)
+ swiotlb_free_from_pool(dev, phys, size, swiotlb_pool);
+ else
+ dma_free_contiguous(dev, page, size);
}
#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 1abd3e6146f4..ac03a6856c2e 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1809,6 +1809,12 @@ bool swiotlb_free(struct device *dev, struct page *page, size_t size)
return true;
}
+void swiotlb_free_from_pool(struct device *dev, phys_addr_t tlb_addr, size_t size,
+ struct io_tlb_pool *pool)
+{
+ swiotlb_release_slots(dev, tlb_addr, pool);
+}
+
static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
struct device *dev)
{
--
2.43.0
^ permalink raw reply related
* [PATCH v5 02/20] [DO NOT MERGE] s390: Expose protected virtualization through cc_platform_has()
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:27 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Halil Pasic,
Matthew Rosato, Jaehoon Kim
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
Protected virtualization guests use memory encryption, so advertise that to
the rest of the kernel through cc_platform_has(CC_ATTR_MEM_ENCRYPT).
s390 already forces DMA mappings to be unencrypted for protected
virtualization guests through force_dma_unencrypted(). Add
ARCH_HAS_CC_PLATFORM and provide the matching cc_platform_has()
implementation
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
Cc: Halil Pasic <pasic@linux.ibm.com>
Cc: Matthew Rosato <mjrosato@linux.ibm.com>
Cc: Jaehoon Kim <jhkim@linux.ibm.com>
---
arch/s390/Kconfig | 1 +
arch/s390/mm/init.c | 14 ++++++++++++++
2 files changed, 15 insertions(+)
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index ecbcbb781e40..9b5e6029e043 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -87,6 +87,7 @@ config S390
select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
select ARCH_HAS_CC_CAN_LINK
+ select ARCH_HAS_CC_PLATFORM
select ARCH_HAS_CPU_FINALIZE_INIT
select ARCH_HAS_CURRENT_STACK_POINTER
select ARCH_HAS_DEBUG_VIRTUAL
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 1f72efc2a579..ad3c6d92b801 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -50,6 +50,7 @@
#include <linux/virtio_anchor.h>
#include <linux/virtio_config.h>
#include <linux/execmem.h>
+#include <linux/cc_platform.h>
pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir");
pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir");
@@ -140,6 +141,19 @@ bool force_dma_unencrypted(struct device *dev)
return is_prot_virt_guest();
}
+
+bool cc_platform_has(enum cc_attr attr)
+{
+ switch (attr) {
+ case CC_ATTR_MEM_ENCRYPT:
+ return is_prot_virt_guest();
+
+ default:
+ return false;
+ }
+}
+EXPORT_SYMBOL_GPL(cc_platform_has);
+
/* protected virtualization */
static void __init pv_init(void)
{
--
2.43.0
^ permalink raw reply related
* [PATCH v5 01/20] [DO NOT MERGE] arm64/coco: Add pKVM as a CC platform
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:27 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86
In-Reply-To: <20260522042815.370873-1-aneesh.kumar@kernel.org>
pKVM does support memory encryption, expose that to the rest of
the kernel through cc_platform_has()
At the moment, all devices inside the guest are emulated which
requires its memory to be shared back to the host (decrypted), so
set force_dma_unencrypted() to always return true.
Signed-off-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
arch/arm64/include/asm/hypervisor.h | 6 ++++++
arch/arm64/include/asm/mem_encrypt.h | 3 ++-
arch/arm64/kernel/rsi.c | 12 ------------
arch/arm64/mm/init.c | 13 +++++++++++++
drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c | 5 +++++
5 files changed, 26 insertions(+), 13 deletions(-)
diff --git a/arch/arm64/include/asm/hypervisor.h b/arch/arm64/include/asm/hypervisor.h
index a12fd897c877..1b0e15f290be 100644
--- a/arch/arm64/include/asm/hypervisor.h
+++ b/arch/arm64/include/asm/hypervisor.h
@@ -10,8 +10,14 @@ void kvm_arm_target_impl_cpu_init(void);
#ifdef CONFIG_ARM_PKVM_GUEST
void pkvm_init_hyp_services(void);
+bool is_protected_kvm_guest(void);
#else
static inline void pkvm_init_hyp_services(void) { };
+
+static inline bool is_protected_kvm_guest(void)
+{
+ return false;
+}
#endif
static inline void kvm_arch_init_hyp_services(void)
diff --git a/arch/arm64/include/asm/mem_encrypt.h b/arch/arm64/include/asm/mem_encrypt.h
index 314b2b52025f..636f45b4d8af 100644
--- a/arch/arm64/include/asm/mem_encrypt.h
+++ b/arch/arm64/include/asm/mem_encrypt.h
@@ -2,6 +2,7 @@
#ifndef __ASM_MEM_ENCRYPT_H
#define __ASM_MEM_ENCRYPT_H
+#include <asm/hypervisor.h>
#include <asm/rsi.h>
struct device;
@@ -20,7 +21,7 @@ int realm_register_memory_enc_ops(void);
static inline bool force_dma_unencrypted(struct device *dev)
{
- return is_realm_world();
+ return is_realm_world() || is_protected_kvm_guest();
}
/*
diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c
index 92160f2e57ff..25ca75ce1a4d 100644
--- a/arch/arm64/kernel/rsi.c
+++ b/arch/arm64/kernel/rsi.c
@@ -7,7 +7,6 @@
#include <linux/memblock.h>
#include <linux/psci.h>
#include <linux/swiotlb.h>
-#include <linux/cc_platform.h>
#include <linux/platform_device.h>
#include <asm/io.h>
@@ -23,17 +22,6 @@ EXPORT_SYMBOL(prot_ns_shared);
DEFINE_STATIC_KEY_FALSE_RO(rsi_present);
EXPORT_SYMBOL(rsi_present);
-bool cc_platform_has(enum cc_attr attr)
-{
- switch (attr) {
- case CC_ATTR_MEM_ENCRYPT:
- return is_realm_world();
- default:
- return false;
- }
-}
-EXPORT_SYMBOL_GPL(cc_platform_has);
-
static bool rsi_version_matches(void)
{
unsigned long ver_lower, ver_higher;
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 97987f850a33..c1b223e7cc8e 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -12,6 +12,7 @@
#include <linux/swap.h>
#include <linux/init.h>
#include <linux/cache.h>
+#include <linux/cc_platform.h>
#include <linux/mman.h>
#include <linux/nodemask.h>
#include <linux/initrd.h>
@@ -36,6 +37,7 @@
#include <asm/boot.h>
#include <asm/fixmap.h>
+#include <asm/hypervisor.h>
#include <asm/kasan.h>
#include <asm/kernel-pgtable.h>
#include <asm/kvm_host.h>
@@ -416,6 +418,17 @@ void dump_mem_limit(void)
}
}
+bool cc_platform_has(enum cc_attr attr)
+{
+ switch (attr) {
+ case CC_ATTR_MEM_ENCRYPT:
+ return is_realm_world() || is_protected_kvm_guest();
+ default:
+ return false;
+ }
+}
+EXPORT_SYMBOL_GPL(cc_platform_has);
+
#ifdef CONFIG_EXECMEM
static u64 module_direct_base __ro_after_init = 0;
static u64 module_plt_base __ro_after_init = 0;
diff --git a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c
index 4230b817a80b..297e6d6019b8 100644
--- a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c
+++ b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c
@@ -95,6 +95,11 @@ static int mmio_guard_ioremap_hook(phys_addr_t phys, size_t size,
return 0;
}
+bool is_protected_kvm_guest(void)
+{
+ return !!pkvm_granule;
+}
+
void pkvm_init_hyp_services(void)
{
int i;
--
2.43.0
^ permalink raw reply related
* [PATCH v5 00/20] dma-mapping: Use DMA_ATTR_CC_SHARED through direct, pool and swiotlb paths
From: Aneesh Kumar K.V (Arm) @ 2026-05-22 4:27 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86
This series propagates DMA_ATTR_CC_SHARED through the dma-direct,
dma-pool, and swiotlb paths so that encrypted and decrypted DMA buffers
are handled consistently.
Today, the direct DMA path mostly relies on force_dma_unencrypted() for
shared/decrypted buffer handling. This series consolidates the
force_dma_unencrypted() checks in the top-level functions and ensures
that the remaining DMA interfaces use DMA attributes to make the correct
decisions.
The series:
- moves swiotlb-backed allocations out of __dma_direct_alloc_pages(),
- propagates DMA_ATTR_CC_SHARED through the dma-direct alloc/free
paths
- teaches the atomic DMA pools to track encrypted versus decrypted
state
- tracks swiotlb pool encryption state and enforces strict pool
selection
- centralizes encrypted/decrypted pgprot handling in dma_pgprot() using
DMA attributes
- passes DMA attributes down to dma_capable() so capability checks can
validate whether the selected DMA address encoding matches
DMA_ATTR_CC_SHARED
- makes dma_direct_map_phys() choose the DMA address encoding from
DMA_ATTR_CC_SHARED and fall back to swiotlb when a shared DMA request
cannot use the direct mapping, which lets arm64 and x86 CCA guests stop
relying on SWIOTLB_FORCE for DMA mappings
- use the selected swiotlb pool state to derive the returned DMA
address.
Changes since v4:
https://lore.kernel.org/all/20260512090408.794195-1-aneesh.kumar@kernel.org
* Add new patches based on Sashiko review:
swiotlb: Preserve allocation virtual address for dynamic pools
dma: free atomic pool pages by physical address
dma: swiotlb: handle set_memory_decrypted() failures
dma: swiotlb: free dynamic pools from process context
iommu/dma: Check atomic pool allocation result directly
* Include pKVM and s390 changes as dependent patches. These are not yet
ready to merge and are waiting for subsystem testing feedback.
* Drop the AMD GART patch because it requires wider testing.
* Update swiotlb_tbl_map_single() to take attrs by reference.
* Switch swiotlb_free() to use rcu_work.
* Avoid calling swiotlb_find_pool() multiple times in the free path.
* Make DMA_ATTR_MMIO imply DMA_ATTR_CC_SHARED for devices requiring unencrypted DMA.
Changes from v3:
https://lore.kernel.org/all/20260427055509.898190-1-aneesh.kumar@kernel.org
* Handle DMA_ATTR_MMIO correctly in dma_direct_map_phys()
* Address most of sashiko review
* Rebase to latest kernel
* drop SWIOTLB_FORCE for s390 and powerpc secure guest.
Changes from v2:
https://lore.kernel.org/all/20260420061415.3650870-1-aneesh.kumar@kernel.org
* pass attrs to dma_capable() and update direct, swiotlb, Xen swiotlb, and
x86 GART paths so the capability checks see the DMA address attr value
DMA_ATTR_CC_SHARED.
* rework dma_direct_map_phys() so DMA_ATTR_CC_SHARED selects
phys_to_dma_unencrypted() while the default path uses
phys_to_dma_encrypted(), with swiotlb fallback when the requested
shared/private state cannot be satisfied by a direct DMA address.
* stop relying on SWIOTLB_FORCE for arm64 and x86 CC guest DMA mappings;
swiotlb is still enabled there, but shared mappings is now selected
through the generic dma_direct_map_phys()/dma_capable() decision instead
of a global force-bounce flag.
Changes from v1:
https://lore.kernel.org/all/20260417085900.3062416-1-aneesh.kumar@kernel.org
* rebased to latest kernel (change from DMA_ATTR_CC_DECRYPTED -> DMA_ATTR_CC_SHARED)
* update the alloc path so DMA_ATTR_CC_SHARED is not a caller-visible attribute.
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Marek Szyprowski <m.szyprowski@samsung.com>
Cc: Will Deacon <will@kernel.org>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Steven Price <steven.price@arm.com>
Cc: Suzuki K Poulose <Suzuki.Poulose@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Jiri Pirko <jiri@resnulli.us>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: Mostafa Saleh <smostafa@google.com>
Cc: Petr Tesarik <ptesarik@suse.com>
Cc: Alexey Kardashevskiy <aik@amd.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Xu Yilun <yilun.xu@linux.intel.com>
Cc: linuxppc-dev@lists.ozlabs.org
Cc: linux-s390@vger.kernel.org
Cc: Madhavan Srinivasan <maddy@linux.ibm.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: "Christophe Leroy (CS GROUP)" <chleroy@kernel.org>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: x86@kernel.org
Aneesh Kumar K.V (Arm) (20):
[DO NOT MERGE] arm64/coco: Add pKVM as a CC platform
[DO NOT MERGE] s390: Expose protected virtualization through
cc_platform_has()
dma-direct: swiotlb: handle swiotlb alloc/free outside
__dma_direct_alloc_pages
dma-direct: use DMA_ATTR_CC_SHARED in alloc/free paths
dma-pool: track decrypted atomic pools and select them via attrs
dma: swiotlb: pass mapping attributes by reference
dma: swiotlb: track pool encryption state and honor DMA_ATTR_CC_SHARED
dma-mapping: make dma_pgprot() honor DMA_ATTR_CC_SHARED
dma-direct: pass attrs to dma_capable() for DMA_ATTR_CC_SHARED checks
dma-direct: make dma_direct_map_phys() honor DMA_ATTR_CC_SHARED
dma-direct: set decrypted flag for remapped DMA allocations
dma-direct: select DMA address encoding from DMA_ATTR_CC_SHARED
dma-pool: fix page leak in atomic_pool_expand() cleanup
dma-direct: rename ret to cpu_addr in alloc helpers
dma-direct: return struct page from dma_direct_alloc_from_pool()
iommu/dma: Check atomic pool allocation result directly
dma: swiotlb: free dynamic pools from process context
dma: swiotlb: handle set_memory_decrypted() failures
dma: free atomic pool pages by physical address
swiotlb: Preserve allocation virtual address for dynamic pools
arch/arm64/include/asm/hypervisor.h | 6 +
arch/arm64/include/asm/mem_encrypt.h | 3 +-
arch/arm64/kernel/rsi.c | 12 -
arch/arm64/mm/init.c | 17 +-
arch/powerpc/platforms/pseries/svm.c | 2 +-
arch/s390/Kconfig | 1 +
arch/s390/mm/init.c | 16 +-
arch/x86/kernel/amd_gart_64.c | 30 +-
arch/x86/kernel/pci-dma.c | 4 +-
drivers/iommu/dma-iommu.c | 15 +-
drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c | 5 +
drivers/xen/swiotlb-xen.c | 8 +-
include/linux/dma-direct.h | 20 +-
include/linux/dma-map-ops.h | 3 +-
include/linux/swiotlb.h | 20 +-
kernel/dma/direct.c | 275 +++++++++++++-----
kernel/dma/direct.h | 47 +--
kernel/dma/mapping.c | 16 +-
kernel/dma/pool.c | 221 ++++++++++----
kernel/dma/swiotlb.c | 270 +++++++++++++----
20 files changed, 717 insertions(+), 274 deletions(-)
base-commit: 50897c955902c93ae71c38698abb910525ebdc89
--
2.43.0
^ permalink raw reply
* [RFC PATCH 15/15] x86/virt/tdx: Enable TDX Quoting extension
From: Xu Yilun @ 2026-05-22 3:41 UTC (permalink / raw)
To: kas, djbw, rick.p.edgecombe, x86, peter.fang
Cc: linux-coco, linux-kernel, kvm, sohil.mehta, yilun.xu, yilun.xu,
baolu.lu, zhenzhong.duan, xiaoyao.li
In-Reply-To: <20260522034128.3144354-1-yilun.xu@linux.intel.com>
From: Peter Fang <peter.fang@intel.com>
Enable the TDX Quoting feature via TDH.SYS.CONFIG when supported by the
TDX module.
The TDX Quoting extension generates TDX attestation Quotes via a
SEAMCALL, without using a discrete Quoting engine.
TDX Module supports add-on TDX features (e.g. TDX Quoting & TDX Module
Extensions) that should be manually enabled by host. It extends
TDH.SYS.CONFIG for host to choose to enable them on bootup.
Call TDH.SYS.CONFIG with a new bitmap input parameter to specify which
features to enable. The bitmap uses the same definitions as
TDX_FEATURES0. But note not all bits in TDX_FEATURES0 are valid for
configuration, e.g. TDX Module Extensions is a service that supports TDX
Quoting, it is implicitly enabled when TDX Quoting is enabled. Setting
TDX_FEATURES0_EXT in the bitmap has no effect.
TDX Module advances the version of TDH.SYS.CONFIG for the change, so
use the latest version (v1) for add-on feature enabling. But supporting
existing Modules which only support v0 is still necessary until they are
deprecated. In fact, it is unlikely that TDH.SYS.CONFIG ever needs to
change again and the code would stay in v1. So there is little value
in worrying about deprecating v0 to save a couple lines of code in 5-7
years when these original TDX platforms sunset.
TDX Module updates global metadata when add-on features are enabled.
Host should update the cached tdx_sysinfo to reflect these changes.
Co-developed-by: Xu Yilun <yilun.xu@linux.intel.com>
Signed-off-by: Xu Yilun <yilun.xu@linux.intel.com>
Signed-off-by: Peter Fang <peter.fang@intel.com>
---
arch/x86/virt/vmx/tdx/tdx.h | 4 +++-
arch/x86/virt/vmx/tdx/tdx.c | 24 ++++++++++++++++++++++--
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h
index 10aff23cd01f..524a14c01aa6 100644
--- a/arch/x86/virt/vmx/tdx/tdx.h
+++ b/arch/x86/virt/vmx/tdx/tdx.h
@@ -58,7 +58,8 @@
#define TDH_PHYMEM_CACHE_WB 40
#define TDH_PHYMEM_PAGE_WBINVD 41
#define TDH_VP_WR 43
-#define TDH_SYS_CONFIG 45
+#define TDH_SYS_CONFIG_V0 45
+#define TDH_SYS_CONFIG SEAMCALL_LEAF_VER(TDH_SYS_CONFIG_V0, 1)
#define TDH_EXT_INIT 60
#define TDH_EXT_MEM_ADD 61
#define TDH_SYS_DISABLE 69
@@ -97,6 +98,7 @@ struct tdmr_info {
/* Bit definitions of TDX_FEATURES0 metadata field */
#define TDX_FEATURES0_NO_RBP_MOD BIT(18)
#define TDX_FEATURES0_EXT BIT_ULL(39)
+#define TDX_FEATURES0_QUOTE BIT_ULL(50)
/*
* Do not put any hardware-defined TDX structure representations below
diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
index f7600f930c6e..86e5b7ad19b3 100644
--- a/arch/x86/virt/vmx/tdx/tdx.c
+++ b/arch/x86/virt/vmx/tdx/tdx.c
@@ -1049,6 +1049,7 @@ static __init int construct_tdmrs(struct list_head *tmb_list,
static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,
u64 global_keyid)
{
+ u64 seamcall_fn = TDH_SYS_CONFIG_V0;
struct tdx_module_args args = {};
u64 *tdmr_pa_array;
size_t array_sz;
@@ -1074,8 +1075,22 @@ static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,
args.rcx = __pa(tdmr_pa_array);
args.rdx = tdmr_list->nr_consumed_tdmrs;
args.r8 = global_keyid;
- ret = seamcall_prerr(TDH_SYS_CONFIG, &args);
+ if (tdx_sysinfo.features.tdx_features0 & TDX_FEATURES0_QUOTE) {
+ args.r9 |= TDX_FEATURES0_QUOTE;
+ /* These parameters require version >= 1 */
+ seamcall_fn = TDH_SYS_CONFIG;
+ }
+
+ ret = seamcall_prerr(seamcall_fn, &args);
+ if (ret)
+ goto free_tdmr;
+
+ /* enabling TDX Quoting may change tdx_sysinfo, update it */
+ if (tdx_sysinfo.features.tdx_features0 & TDX_FEATURES0_QUOTE)
+ ret = get_tdx_sys_info(&tdx_sysinfo);
+
+free_tdmr:
/* Free the array as it is not required anymore. */
kfree(tdmr_pa_array);
@@ -1384,12 +1399,17 @@ static void tdx_quote_init(void)
unsigned int nr_quote_pages;
u64 r;
+ if (!(tdx_sysinfo.features.tdx_features0 & TDX_FEATURES0_QUOTE))
+ return;
+
do {
r = seamcall(TDH_QUOTE_INIT, &args);
} while (r == TDX_INTERRUPTED_RESUMABLE);
- if (r)
+ if (r) {
+ pr_err("Failed to enable quoting extension: 0x%llx\n", r);
return;
+ }
/* Quoting metadata is valid only after initialization */
if (get_tdx_sys_info_quote(&tdx_sysinfo.quote))
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 14/15] x86/virt/tdx: Embed version info in SEAMCALL leaf function definitions
From: Xu Yilun @ 2026-05-22 3:41 UTC (permalink / raw)
To: kas, djbw, rick.p.edgecombe, x86, peter.fang
Cc: linux-coco, linux-kernel, kvm, sohil.mehta, yilun.xu, yilun.xu,
baolu.lu, zhenzhong.duan, xiaoyao.li
In-Reply-To: <20260522034128.3144354-1-yilun.xu@linux.intel.com>
Embed version information in SEAMCALL leaf function definitions rather
than let the caller open code them. For now, only TDH.VP.INIT is
involved.
Don't bother the caller to choose the SEAMCALL version if unnecessary.
New version SEAMCALLs are guaranteed to be backward compatible, so
ideally kernel doesn't need to keep version history and only uses the
latest version SEAMCALLs.
The concern is some old TDX Modules don't recognize new version
SEAMCALLs. Multiple SEAMCALL versions co-exist when kernel should
support these old Modules. As time goes by, the old Modules deprecate
and old version SEAMCALL definitions should disappear.
The old TDX Modules that only support TDH.VP.INIT v0 are all deprecated,
so only provide the latest (v1) definition.
Signed-off-by: Xu Yilun <yilun.xu@linux.intel.com>
---
arch/x86/virt/vmx/tdx/tdx.h | 23 ++++++++++++++---------
arch/x86/virt/vmx/tdx/tdx.c | 4 ++--
2 files changed, 16 insertions(+), 11 deletions(-)
diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h
index 01a7d7d8ada9..10aff23cd01f 100644
--- a/arch/x86/virt/vmx/tdx/tdx.h
+++ b/arch/x86/virt/vmx/tdx/tdx.h
@@ -2,6 +2,7 @@
#ifndef _X86_VIRT_TDX_H
#define _X86_VIRT_TDX_H
+#include <linux/bitfield.h>
#include <linux/bits.h>
/*
@@ -11,6 +12,18 @@
* architectural definitions come first.
*/
+/*
+ * SEAMCALL leaf:
+ *
+ * Bit 15:0 Leaf number
+ * Bit 23:16 Version number
+ */
+#define SEAMCALL_LEAF GENMASK(15, 0)
+#define SEAMCALL_VER GENMASK(23, 16)
+
+#define SEAMCALL_LEAF_VER(l, v) (FIELD_PREP(SEAMCALL_LEAF, l) | \
+ FIELD_PREP(SEAMCALL_VER, v))
+
/*
* TDX module SEAMCALL leaf functions
*/
@@ -31,7 +44,7 @@
#define TDH_VP_CREATE 10
#define TDH_MNG_KEY_FREEID 20
#define TDH_MNG_INIT 21
-#define TDH_VP_INIT 22
+#define TDH_VP_INIT SEAMCALL_LEAF_VER(22, 1)
#define TDH_PHYMEM_PAGE_RDMD 24
#define TDH_VP_RD 26
#define TDH_PHYMEM_PAGE_RECLAIM 28
@@ -52,14 +65,6 @@
#define TDH_QUOTE_GET 98
#define TDH_QUOTE_INIT 100
-/*
- * SEAMCALL leaf:
- *
- * Bit 15:0 Leaf number
- * Bit 23:16 Version number
- */
-#define TDX_VERSION_SHIFT 16
-
/* TDX page types */
#define PT_NDA 0x0
#define PT_RSVD 0x1
diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
index 821f677e9a86..f7600f930c6e 100644
--- a/arch/x86/virt/vmx/tdx/tdx.c
+++ b/arch/x86/virt/vmx/tdx/tdx.c
@@ -2217,8 +2217,8 @@ u64 tdh_vp_init(struct tdx_vp *vp, u64 initial_rcx, u32 x2apicid)
.r8 = x2apicid,
};
- /* apicid requires version == 1. */
- return seamcall(TDH_VP_INIT | (1ULL << TDX_VERSION_SHIFT), &args);
+ /* apicid requires version == 1. See TDH_VP_INIT definition.*/
+ return seamcall(TDH_VP_INIT, &args);
}
EXPORT_SYMBOL_FOR_KVM(tdh_vp_init);
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 13/15] KVM: TDX: Support event-notify interrupts only with userspace quoting
From: Xu Yilun @ 2026-05-22 3:41 UTC (permalink / raw)
To: kas, djbw, rick.p.edgecombe, x86, peter.fang
Cc: linux-coco, linux-kernel, kvm, sohil.mehta, yilun.xu, yilun.xu,
baolu.lu, zhenzhong.duan, xiaoyao.li
In-Reply-To: <20260522034128.3144354-1-yilun.xu@linux.intel.com>
From: Peter Fang <peter.fang@intel.com>
Tie userspace SetupEventNotifyInterrupt support to userspace Quote
generation. Delivering event-notify interrupts via userspace breaks if
KVM never exits to userspace in the first place.
No known guest currently requires event-notify interrupt support, so
defer adding in-kernel support for now. Linux TDX guests use polling
only.
Update the KVM API Documentation to reflect the change.
Signed-off-by: Peter Fang <peter.fang@intel.com>
Signed-off-by: Xu Yilun <yilun.xu@linux.intel.com>
---
Documentation/virt/kvm/api.rst | 8 +++++++-
arch/x86/kvm/vmx/tdx.c | 20 +++++++++++++++++---
2 files changed, 24 insertions(+), 4 deletions(-)
diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst
index 52bbbb553ce1..8a02745a36ee 100644
--- a/Documentation/virt/kvm/api.rst
+++ b/Documentation/virt/kvm/api.rst
@@ -7335,6 +7335,9 @@ inputs and outputs of the TDVMCALL. Currently the following values of
queued successfully, the TDX guest can poll the status field in the
shared-memory area to check whether the Quote generation is completed or
not. When completed, the generated Quote is returned via the same buffer.
+ If the host kernel generates Quotes through the TDX Quoting service provided
+ by the TDX module, KVM processes the GetQuote request and it will not appear
+ in userspace. KVM only supports version 1 of the GetQuote request.
* ``TDVMCALL_GET_TD_VM_CALL_INFO``: the guest has requested the support
status of TDVMCALLs. The output values for the given leaf should be
@@ -7342,7 +7345,10 @@ inputs and outputs of the TDVMCALL. Currently the following values of
field of the union.
* ``TDVMCALL_SETUP_EVENT_NOTIFY_INTERRUPT``: the guest has requested to
- set up a notification interrupt for vector ``vector``.
+ set up a notification interrupt for vector ``vector``. Since this TDVMCALL
+ is used to optimize ``TDVMCALL_GET_QUOTE``, KVM disables this support in
+ userspace VMM if ``TDVMCALL_GET_QUOTE`` is completely handled in the kernel.
+ KVM may add kernel support for this in the future.
KVM may add support for more values in the future that may cause a userspace
exit, even without calls to ``KVM_ENABLE_CAP`` or similar. In this case,
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index bade046da5a1..5aebbec7fa6e 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -185,7 +185,7 @@ static void td_init_cpuid_entry2(struct kvm_cpuid_entry2 *entry, unsigned char i
tdx_clear_unsupported_cpuid(entry);
}
-#define TDVMCALLINFO_SETUP_EVENT_NOTIFY_INTERRUPT BIT(1)
+#define TDVMCALLINFO_SETUP_EVENT_NOTIFY_INTERRUPT BIT_ULL(1)
static int init_kvm_tdx_caps(const struct tdx_sys_info_td_conf *td_conf,
struct kvm_tdx_capabilities *caps)
@@ -202,8 +202,15 @@ static int init_kvm_tdx_caps(const struct tdx_sys_info_td_conf *td_conf,
caps->cpuid.nent = td_conf->num_cpuid_config;
- caps->user_tdvmcallinfo_1_r11 =
- TDVMCALLINFO_SETUP_EVENT_NOTIFY_INTERRUPT;
+ /*
+ * Don't advertise userspace event-notify interrupt support if TDX
+ * quoting service is enabled, as quote generation will be done entirely
+ * in the kernel. Support in the kernel can be added later if needed.
+ */
+ if (!tdx_quote_enabled()) {
+ caps->user_tdvmcallinfo_1_r11 |=
+ TDVMCALLINFO_SETUP_EVENT_NOTIFY_INTERRUPT;
+ }
for (i = 0; i < td_conf->num_cpuid_config; i++)
td_init_cpuid_entry2(&caps->cpuid.entries[i], i);
@@ -1684,9 +1691,16 @@ static int tdx_get_quote(struct kvm_vcpu *vcpu)
static int tdx_setup_event_notify_interrupt(struct kvm_vcpu *vcpu)
{
+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
struct vcpu_tdx *tdx = to_tdx(vcpu);
u64 vector = tdx->vp_enter_args.r12;
+ /* See init_kvm_tdx_caps() for comments */
+ if (kvm_tdx->get_quote_in_kernel) {
+ tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_SUBFUNC_UNSUPPORTED);
+ return 1;
+ }
+
if (vector < 32 || vector > 255) {
tdvmcall_set_return_code(vcpu, TDVMCALL_STATUS_INVALID_OPERAND);
return 1;
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 12/15] KVM: TDX: Add in-kernel Quote generation
From: Xu Yilun @ 2026-05-22 3:41 UTC (permalink / raw)
To: kas, djbw, rick.p.edgecombe, x86, peter.fang
Cc: linux-coco, linux-kernel, kvm, sohil.mehta, yilun.xu, yilun.xu,
baolu.lu, zhenzhong.duan, xiaoyao.li
In-Reply-To: <20260522034128.3144354-1-yilun.xu@linux.intel.com>
From: Peter Fang <peter.fang@intel.com>
Provide an in-kernel path for TDX Quote generation when handling
TDG.VP.VMCALL<GetQuote>, without requiring an exit to userspace.
Use the core TDX API when the TDX Quoting extension is available. For
simplicity, each KVM guest checks for availability only once during
initialization. KVM does not handle Quoting service disruptions.
Signed-off-by: Peter Fang <peter.fang@intel.com>
Signed-off-by: Xu Yilun <yilun.xu@linux.intel.com>
---
arch/x86/include/asm/tdx.h | 9 +++
arch/x86/kvm/vmx/tdx.h | 6 ++
arch/x86/kvm/vmx/tdx.c | 135 ++++++++++++++++++++++++++++++++++++-
virt/kvm/kvm_main.c | 1 +
4 files changed, 150 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index 945e6817abb2..5863d6748100 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -115,6 +115,15 @@ struct tdx_quote_req {
u32 out_len;
u8 data[];
};
+
+#define TDX_QUOTE_REQ_HDR_SIZE (offsetof(struct tdx_quote_req, data))
+
+/*
+ * TDG.VP.VMCALL<GetQuote> Status Codes
+ */
+#define TDX_QUOTE_STATUS_SUCCESS 0x0000000000000000ULL
+#define TDX_QUOTE_STATUS_ERROR 0x8000000000000000ULL
+#define TDX_QUOTE_STATUS_UNAVAILABLE 0x8000000000000001ULL
#endif /* CONFIG_INTEL_TDX_GUEST || CONFIG_KVM_INTEL_TDX */
#ifdef CONFIG_INTEL_TDX_HOST
diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h
index ac8323a68b16..18c93e80c0ec 100644
--- a/arch/x86/kvm/vmx/tdx.h
+++ b/arch/x86/kvm/vmx/tdx.h
@@ -47,6 +47,12 @@ struct kvm_tdx {
* Set/unset is protected with kvm->mmu_lock.
*/
bool wait_for_sept_zap;
+
+ /*
+ * Whether to get TDX quote directly in kernel, without exiting to
+ * userspace.
+ */
+ bool get_quote_in_kernel;
};
/* TDX module vCPU states */
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index 9f7c39e0d4b5..bade046da5a1 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -1538,11 +1538,133 @@ static int tdx_get_quote_user(struct kvm_vcpu *vcpu, u64 gpa, u64 size)
return 0;
}
+static bool write_quote_status_to_guest(struct kvm_vcpu *vcpu, u64 status,
+ gpa_t gpa)
+{
+ if (kvm_vcpu_write_guest(vcpu,
+ gpa + offsetof(struct tdx_quote_req, status),
+ &status, sizeof(status)))
+ return false;
+
+ return true;
+}
+
+static bool write_quote_to_guest(struct kvm_vcpu *vcpu, void *quote_data,
+ u32 quote_len, gpa_t gpa)
+{
+ if (kvm_vcpu_write_guest(vcpu,
+ gpa + TDX_QUOTE_REQ_HDR_SIZE,
+ quote_data, quote_len))
+ return false;
+
+ if (kvm_vcpu_write_guest(vcpu,
+ gpa + offsetof(struct tdx_quote_req, out_len),
+ "e_len, sizeof(quote_len)))
+ return false;
+
+ return true;
+}
+
+static u64 __get_quote_kernel(struct kvm_vcpu *vcpu, struct tdx_quote_req *req,
+ size_t req_len, gpa_t req_gpa, size_t total_len)
+{
+ struct tdx_td *td = &to_kvm_tdx(vcpu->kvm)->td;
+
+ /* Only support version 1 as defined in the GHCI spec */
+ if (req->version != 1)
+ return TDX_QUOTE_STATUS_ERROR;
+
+ if ((size_t)req->in_len + TDX_QUOTE_REQ_HDR_SIZE > req_len)
+ return TDX_QUOTE_STATUS_ERROR;
+
+ /* The caller frees the quote data */
+ void *quote_data __free(kvfree) =
+ tdx_quote_generate(td, req->data, req->in_len, &req->out_len);
+
+ if (!quote_data)
+ return TDX_QUOTE_STATUS_UNAVAILABLE;
+
+ if ((size_t)req->out_len + TDX_QUOTE_REQ_HDR_SIZE > total_len)
+ return TDX_QUOTE_STATUS_ERROR;
+
+ if (!write_quote_to_guest(vcpu, quote_data, req->out_len, req_gpa))
+ return TDX_QUOTE_STATUS_ERROR;
+
+ return TDX_QUOTE_STATUS_SUCCESS;
+}
+
+static u64 tdx_get_quote_check_args(struct kvm_vcpu *vcpu, u64 gpa, u64 size)
+{
+ gfn_t gfn_start, gfn_end;
+ u64 end;
+
+ if (!size)
+ return TDVMCALL_STATUS_INVALID_OPERAND;
+
+ if (!PAGE_ALIGNED(gpa) || !PAGE_ALIGNED(size))
+ return TDVMCALL_STATUS_ALIGN_ERROR;
+
+ if (check_add_overflow(gpa, size, &end))
+ return TDVMCALL_STATUS_INVALID_OPERAND;
+
+ gfn_start = gpa_to_gfn(gpa);
+ gfn_end = gpa_to_gfn(end);
+
+ /*
+ * Reject if the guest didn't explicitly convert its quote pages to
+ * shared.
+ */
+ if (!kvm_range_has_memory_attributes(vcpu->kvm, gfn_start, gfn_end,
+ KVM_MEMORY_ATTRIBUTE_PRIVATE, 0))
+ return TDVMCALL_STATUS_INVALID_OPERAND;
+
+ return TDVMCALL_STATUS_SUCCESS;
+}
+
+static int tdx_get_quote_kernel(struct kvm_vcpu *vcpu, u64 gpa, u64 size)
+{
+ void *first_page = NULL;
+ u64 err, qerr;
+
+ err = tdx_get_quote_check_args(vcpu, gpa, size);
+ if (err != TDVMCALL_STATUS_SUCCESS)
+ goto out;
+
+ err = TDVMCALL_STATUS_INVALID_OPERAND;
+
+ first_page = kmalloc(PAGE_SIZE, GFP_KERNEL);
+ if (!first_page)
+ goto out;
+
+ /*
+ * Read the first GetQuote page for its header + in_data. The check
+ * above ensures that this GetQuote message is at least one page in
+ * size. in_data spanning more than a page is not supported.
+ */
+ if (kvm_vcpu_read_guest(vcpu, gpa, first_page, PAGE_SIZE))
+ goto out;
+
+ qerr = __get_quote_kernel(vcpu, first_page, PAGE_SIZE,
+ (gpa_t)gpa, size);
+
+ if (write_quote_status_to_guest(vcpu, qerr, (gpa_t)gpa) &&
+ qerr == TDX_QUOTE_STATUS_SUCCESS)
+ err = TDVMCALL_STATUS_SUCCESS;
+
+out:
+ kfree(first_page);
+ tdvmcall_set_return_code(vcpu, err);
+
+ return 1;
+}
+
static int tdx_get_quote(struct kvm_vcpu *vcpu)
{
+ struct kvm_tdx *kvm_tdx = to_kvm_tdx(vcpu->kvm);
struct vcpu_tdx *tdx = to_tdx(vcpu);
u64 gpa = tdx->vp_enter_args.r12;
u64 size = tdx->vp_enter_args.r13;
+ int ret;
/* The gpa of buffer must have shared bit set. */
if (vt_is_tdx_private_gpa(vcpu->kvm, gpa)) {
@@ -1552,7 +1674,12 @@ static int tdx_get_quote(struct kvm_vcpu *vcpu)
gpa &= ~gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm));
- return tdx_get_quote_user(vcpu, gpa, size);
+ if (kvm_tdx->get_quote_in_kernel)
+ ret = tdx_get_quote_kernel(vcpu, gpa, size);
+ else
+ ret = tdx_get_quote_user(vcpu, gpa, size);
+
+ return ret;
}
static int tdx_setup_event_notify_interrupt(struct kvm_vcpu *vcpu)
@@ -2751,6 +2878,12 @@ static int tdx_td_init(struct kvm *kvm, struct kvm_tdx_cmd *cmd)
else
kvm->arch.gfn_direct_bits = TDX_SHARED_BIT_PWL_4;
+ /*
+ * Check only once at TD creation. If the quoting service gets disrupted
+ * during TD runtime, let the user handle it.
+ */
+ kvm_tdx->get_quote_in_kernel = tdx_quote_enabled();
+
kvm_tdx->state = TD_STATE_INITIALIZED;
out:
/* kfree() accepts NULL. */
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 89489996fbc1..599f88a13071 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -2461,6 +2461,7 @@ bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end,
return true;
}
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_range_has_memory_attributes);
static __always_inline void kvm_handle_gfn_range(struct kvm *kvm,
struct kvm_mmu_notifier_range *range)
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 11/15] KVM: TDX: Factor out userspace return path from tdx_get_quote()
From: Xu Yilun @ 2026-05-22 3:41 UTC (permalink / raw)
To: kas, djbw, rick.p.edgecombe, x86, peter.fang
Cc: linux-coco, linux-kernel, kvm, sohil.mehta, yilun.xu, yilun.xu,
baolu.lu, zhenzhong.duan, xiaoyao.li
In-Reply-To: <20260522034128.3144354-1-yilun.xu@linux.intel.com>
From: Peter Fang <peter.fang@intel.com>
Separate the logic that returns GetQuote to userspace so that
tdx_get_quote() can be extended to support in-kernel quote generation.
No functional change intended.
Signed-off-by: Peter Fang <peter.fang@intel.com>
Signed-off-by: Xu Yilun <yilun.xu@linux.intel.com>
---
arch/x86/kvm/vmx/tdx.c | 25 ++++++++++++++++---------
1 file changed, 16 insertions(+), 9 deletions(-)
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index ed12805bbb44..9f7c39e0d4b5 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -1524,6 +1524,20 @@ static int tdx_complete_simple(struct kvm_vcpu *vcpu)
return 1;
}
+static int tdx_get_quote_user(struct kvm_vcpu *vcpu, u64 gpa, u64 size)
+{
+ vcpu->run->exit_reason = KVM_EXIT_TDX;
+ vcpu->run->tdx.flags = 0;
+ vcpu->run->tdx.nr = TDVMCALL_GET_QUOTE;
+ vcpu->run->tdx.get_quote.ret = TDVMCALL_STATUS_SUBFUNC_UNSUPPORTED;
+ vcpu->run->tdx.get_quote.gpa = gpa;
+ vcpu->run->tdx.get_quote.size = size;
+
+ vcpu->arch.complete_userspace_io = tdx_complete_simple;
+
+ return 0;
+}
+
static int tdx_get_quote(struct kvm_vcpu *vcpu)
{
struct vcpu_tdx *tdx = to_tdx(vcpu);
@@ -1536,16 +1550,9 @@ static int tdx_get_quote(struct kvm_vcpu *vcpu)
return 1;
}
- vcpu->run->exit_reason = KVM_EXIT_TDX;
- vcpu->run->tdx.flags = 0;
- vcpu->run->tdx.nr = TDVMCALL_GET_QUOTE;
- vcpu->run->tdx.get_quote.ret = TDVMCALL_STATUS_SUBFUNC_UNSUPPORTED;
- vcpu->run->tdx.get_quote.gpa = gpa & ~gfn_to_gpa(kvm_gfn_direct_bits(tdx->vcpu.kvm));
- vcpu->run->tdx.get_quote.size = size;
-
- vcpu->arch.complete_userspace_io = tdx_complete_simple;
+ gpa &= ~gfn_to_gpa(kvm_gfn_direct_bits(vcpu->kvm));
- return 0;
+ return tdx_get_quote_user(vcpu, gpa, size);
}
static int tdx_setup_event_notify_interrupt(struct kvm_vcpu *vcpu)
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 10/15] x86/tdx: Move and rename Quote request structure
From: Xu Yilun @ 2026-05-22 3:41 UTC (permalink / raw)
To: kas, djbw, rick.p.edgecombe, x86, peter.fang
Cc: linux-coco, linux-kernel, kvm, sohil.mehta, yilun.xu, yilun.xu,
baolu.lu, zhenzhong.duan, xiaoyao.li
In-Reply-To: <20260522034128.3144354-1-yilun.xu@linux.intel.com>
From: Peter Fang <peter.fang@intel.com>
struct tdx_quote_buf is currently used only by the guest, but the Quote
buffer format will also be needed by the host for in-kernel Quote
generation. Move the definition to tdx.h so it can be shared by both.
Rename the struct to tdx_quote_req to better reflect its purpose.
Signed-off-by: Peter Fang <peter.fang@intel.com>
Signed-off-by: Xu Yilun <yilun.xu@linux.intel.com>
---
arch/x86/include/asm/tdx.h | 21 +++++++++++++++++++++
drivers/virt/coco/tdx-guest/tdx-guest.c | 25 +++----------------------
2 files changed, 24 insertions(+), 22 deletions(-)
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index bc512a00a0d0..945e6817abb2 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -96,6 +96,27 @@ static inline long tdx_kvm_hypercall(unsigned int nr, unsigned long p1,
}
#endif /* CONFIG_INTEL_TDX_GUEST && CONFIG_KVM_GUEST */
+#if defined(CONFIG_INTEL_TDX_GUEST) || defined(CONFIG_KVM_INTEL_TDX)
+/* struct tdx_quote_req: Format of Quote request message.
+ * @version: Quote format version, filled by TD.
+ * @status: Status code of Quote request, filled by VMM.
+ * @in_len: Length of TDREPORT, filled by TD.
+ * @out_len: Length of Quote data, filled by VMM.
+ * @data: Quote data on output or TDREPORT on input.
+ *
+ * More details of Quote request message can be found in TDX
+ * Guest-Host Communication Interface (GHCI) for Intel TDX 1.0,
+ * section titled "TDG.VP.VMCALL<GetQuote>"
+ */
+struct tdx_quote_req {
+ u64 version;
+ u64 status;
+ u32 in_len;
+ u32 out_len;
+ u8 data[];
+};
+#endif /* CONFIG_INTEL_TDX_GUEST || CONFIG_KVM_INTEL_TDX */
+
#ifdef CONFIG_INTEL_TDX_HOST
u64 __seamcall(u64 fn, struct tdx_module_args *args);
u64 __seamcall_ret(u64 fn, struct tdx_module_args *args);
diff --git a/drivers/virt/coco/tdx-guest/tdx-guest.c b/drivers/virt/coco/tdx-guest/tdx-guest.c
index a9ecc46df187..d0ddbbc98fb8 100644
--- a/drivers/virt/coco/tdx-guest/tdx-guest.c
+++ b/drivers/virt/coco/tdx-guest/tdx-guest.c
@@ -171,26 +171,7 @@ static void tdx_mr_deinit(const struct attribute_group *mr_grp)
#define GET_QUOTE_SUCCESS 0
#define GET_QUOTE_IN_FLIGHT 0xffffffffffffffff
-#define TDX_QUOTE_MAX_LEN (GET_QUOTE_BUF_SIZE - sizeof(struct tdx_quote_buf))
-
-/* struct tdx_quote_buf: Format of Quote request buffer.
- * @version: Quote format version, filled by TD.
- * @status: Status code of Quote request, filled by VMM.
- * @in_len: Length of TDREPORT, filled by TD.
- * @out_len: Length of Quote data, filled by VMM.
- * @data: Quote data on output or TDREPORT on input.
- *
- * More details of Quote request buffer can be found in TDX
- * Guest-Host Communication Interface (GHCI) for Intel TDX 1.0,
- * section titled "TDG.VP.VMCALL<GetQuote>"
- */
-struct tdx_quote_buf {
- u64 version;
- u64 status;
- u32 in_len;
- u32 out_len;
- u8 data[];
-};
+#define TDX_QUOTE_MAX_LEN (GET_QUOTE_BUF_SIZE - sizeof(struct tdx_quote_req))
/* Quote data buffer */
static void *quote_data;
@@ -250,7 +231,7 @@ static void *alloc_quote_buf(void)
* or error code after processing is complete. So wait till the status
* changes from GET_QUOTE_IN_FLIGHT or the request being timed out.
*/
-static int wait_for_quote_completion(struct tdx_quote_buf *quote_buf, u32 timeout)
+static int wait_for_quote_completion(struct tdx_quote_req *quote_buf, u32 timeout)
{
int i = 0;
@@ -269,7 +250,7 @@ static int wait_for_quote_completion(struct tdx_quote_buf *quote_buf, u32 timeou
static int tdx_report_new_locked(struct tsm_report *report, void *data)
{
u8 *buf;
- struct tdx_quote_buf *quote_buf = quote_data;
+ struct tdx_quote_req *quote_buf = quote_data;
struct tsm_report_desc *desc = &report->desc;
u32 out_len;
int ret;
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 09/15] x86/virt/tdx: Add interface to generate a Quote
From: Xu Yilun @ 2026-05-22 3:41 UTC (permalink / raw)
To: kas, djbw, rick.p.edgecombe, x86, peter.fang
Cc: linux-coco, linux-kernel, kvm, sohil.mehta, yilun.xu, yilun.xu,
baolu.lu, zhenzhong.duan, xiaoyao.li
In-Reply-To: <20260522034128.3144354-1-yilun.xu@linux.intel.com>
From: Peter Fang <peter.fang@intel.com>
Use the TDX Quoting extension's TDH.QUOTE.GET SEAMCALL to generate a
Quote. Since the interface is shared across all KVM instances,
serialize access to the SEAMCALL buffer with a mutex.
Allocate and return a per-call buffer containing the generated Quote so
callers don't need to size the Quote buffer themselves. The caller is
responsible for freeing the returned buffer.
Signed-off-by: Peter Fang <peter.fang@intel.com>
Signed-off-by: Xu Yilun <yilun.xu@linux.intel.com>
---
arch/x86/include/asm/tdx.h | 2 +
arch/x86/virt/vmx/tdx/tdx.h | 1 +
arch/x86/virt/vmx/tdx/tdx.c | 82 +++++++++++++++++++++++++++++++++++++
3 files changed, 85 insertions(+)
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index 7b257088aa1e..bc512a00a0d0 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -177,6 +177,8 @@ struct tdx_vp {
};
bool tdx_quote_enabled(void);
+void *tdx_quote_generate(struct tdx_td *td, void *in_data, u32 in_data_len,
+ u32 *quote_len);
static inline u64 mk_keyed_paddr(u16 hkid, struct page *page)
{
diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h
index 3849f4f9cc78..01a7d7d8ada9 100644
--- a/arch/x86/virt/vmx/tdx/tdx.h
+++ b/arch/x86/virt/vmx/tdx/tdx.h
@@ -49,6 +49,7 @@
#define TDH_EXT_INIT 60
#define TDH_EXT_MEM_ADD 61
#define TDH_SYS_DISABLE 69
+#define TDH_QUOTE_GET 98
#define TDH_QUOTE_INIT 100
/*
diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
index b305fa5aab5c..821f677e9a86 100644
--- a/arch/x86/virt/vmx/tdx/tdx.c
+++ b/arch/x86/virt/vmx/tdx/tdx.c
@@ -62,6 +62,8 @@ static LIST_HEAD(tdx_memlist);
static struct tdx_sys_info tdx_sysinfo __ro_after_init;
static bool tdx_module_initialized __ro_after_init;
+static DEFINE_MUTEX(tdx_quote_lock);
+
static struct quote_data {
void *buf;
u64 buf_len;
@@ -1228,6 +1230,86 @@ bool tdx_quote_enabled(void)
}
EXPORT_SYMBOL_FOR_KVM(tdx_quote_enabled);
+#define QUOTE_ID_MASK GENMASK_U64(47, 32)
+
+static u64 tdx_quote_get(struct tdx_td *td, u64 in_data_pa, u64 in_data_len,
+ u64 hpa_list_pa, u64 total_len, u64 *quote_len)
+{
+ struct tdx_module_args args = {
+ .rcx = tdx_tdr_pa(td),
+ /* Don't bother specifying the quote id */
+ .rdx = QUOTE_ID_MASK & (u64)-1,
+ .r8 = in_data_pa,
+ .r9 = in_data_len,
+ .r10 = hpa_list_pa,
+ .r11 = total_len,
+ };
+ u64 r;
+
+ do {
+ r = seamcall_ret(TDH_QUOTE_GET, &args);
+ } while (r == TDX_INTERRUPTED_RESUMABLE);
+
+ *quote_len = args.rcx;
+
+ return r;
+}
+
+/**
+ * tdx_quote_generate() - Generate a quote for a TD
+ * @td: The TD to generate the quote for.
+ * @in_data: Input data for the quote request.
+ * @in_data_len: Size of the input data in bytes.
+ * @quote_len: Returned size of the generated quote in bytes.
+ *
+ * Use the TDX Quoting extension to generate a TD quote. Pass the input data
+ * through the shared quote buffer and return the quote.
+ *
+ * Return: Newly allocated quote buffer or %NULL on failure.
+ * The caller must free the returned buffer with kvfree().
+ */
+void *tdx_quote_generate(struct tdx_td *td, void *in_data, u32 in_data_len,
+ u32 *quote_len)
+{
+ void *quote_dup = NULL;
+ u64 r, out_len;
+
+ if (!tdx_quote_enabled())
+ return NULL;
+
+ /* TDH.QUOTE.GET expects the input data to fit in a page */
+ if (in_data_len > PAGE_SIZE)
+ return NULL;
+
+ mutex_lock(&tdx_quote_lock);
+
+ /*
+ * Use the first page of the quote buffer for input data. The buffer
+ * must be at least one page in size. @in_data may not be page-aligned,
+ * but TDH.QUOTE.GET expects page-aligned addresses.
+ */
+ memcpy(quote_data.buf, in_data, (size_t)in_data_len);
+
+ r = tdx_quote_get(td, quote_data.hpa_list[0], (u64)in_data_len,
+ quote_data.hpa_list_pa, quote_data.buf_len, &out_len);
+ if (r || !out_len || out_len > quote_data.buf_len)
+ goto out;
+
+ /*
+ * The quote buffer is a shared resource, so use it only for the
+ * SEAMCALL and copy the data out as soon as possible.
+ */
+ quote_dup = kvmemdup(quote_data.buf, out_len, GFP_KERNEL);
+
+out:
+ mutex_unlock(&tdx_quote_lock);
+
+ *quote_len = (u32)out_len;
+
+ return quote_dup;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_quote_generate);
+
#define HPAS_PER_PAGE (PAGE_SIZE / sizeof(u64))
static int tdx_quote_create_buf(unsigned int nr_pages, struct quote_data *qdata)
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 08/15] x86/virt/tdx: Add interface to check Quoting availability
From: Xu Yilun @ 2026-05-22 3:41 UTC (permalink / raw)
To: kas, djbw, rick.p.edgecombe, x86, peter.fang
Cc: linux-coco, linux-kernel, kvm, sohil.mehta, yilun.xu, yilun.xu,
baolu.lu, zhenzhong.duan, xiaoyao.li
In-Reply-To: <20260522034128.3144354-1-yilun.xu@linux.intel.com>
From: Peter Fang <peter.fang@intel.com>
KVM needs to know if the Quoting extension is available to determine
whether userspace must be involved in Quote generation.
Since the Quote buffer is always created during Quoting extension
bringup, checking whether the buffer exists is sufficient.
Signed-off-by: Peter Fang <peter.fang@intel.com>
Signed-off-by: Xu Yilun <yilun.xu@linux.intel.com>
---
arch/x86/include/asm/tdx.h | 2 ++
arch/x86/virt/vmx/tdx/tdx.c | 15 +++++++++++++++
2 files changed, 17 insertions(+)
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index 15eac89b0afb..7b257088aa1e 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -176,6 +176,8 @@ struct tdx_vp {
struct page **tdcx_pages;
};
+bool tdx_quote_enabled(void);
+
static inline u64 mk_keyed_paddr(u16 hkid, struct page *page)
{
u64 ret;
diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
index 9d04293394d7..b305fa5aab5c 100644
--- a/arch/x86/virt/vmx/tdx/tdx.c
+++ b/arch/x86/virt/vmx/tdx/tdx.c
@@ -1213,6 +1213,21 @@ static inline u64 tdx_tdr_pa(struct tdx_td *td)
return page_to_phys(td->tdr_page);
}
+/**
+ * tdx_quote_enabled() - Check whether TDX Quoting extension is available
+ *
+ * Return: %true if the Quoting extension is available, otherwise %false.
+ */
+bool tdx_quote_enabled(void)
+{
+ /*
+ * No need for locking here. The quote buffer is initialized as part of
+ * core TDX bringup, which comes before KVM is ready for userspace.
+ */
+ return !!quote_data.buf;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_quote_enabled);
+
#define HPAS_PER_PAGE (PAGE_SIZE / sizeof(u64))
static int tdx_quote_create_buf(unsigned int nr_pages, struct quote_data *qdata)
--
2.25.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox