* [PATCH 3/5] powerpc/perf: Capture the HTM memory configuration as part of perf data
From: Athira Rajeev @ 2026-07-01 8:38 UTC (permalink / raw)
To: linuxppc-dev, maddy
Cc: linux-perf-users, atrajeev, hbathini, tejas05, venkat88, tshah
In-Reply-To: <20260701083806.79358-1-atrajeev@linux.ibm.com>
H_HTM (Hardware Trace Macro) hypervisor call has capability
to capture SystemMemory Configuration for a system. This
information helps to understand the physical to logical real
address mapping for the logical partitions in the system.
Along with saving HTM trace data, add support to capture
the memory mapping information also using the hcall.
Patch adds support in perf driver to expose HTM memory
configuration as part of perf.data
When monitoring the HTM pmu, auxiliary buffer captures
the "trace" data and SystemMemory Configuration. This
will be post processed later using perf. The size of memory
mapping data captured depends on how large is the system
and how much memory is allocated. To help with relating
and identifying the start of memory mapping data in the
auxiliary buffer, insert two PERF_SAMPLE_RAW records in the
ring buffer. First PERF_SAMPLE_RAW record will mark the
beginning of system memory mapping data in aux buffer. And second
PERF_SAMPLE_RAW record will be written at the end to make the
end of the data in aux buffer and also contains the total size
of the memory map data. These sample raw records
will be used during post processing in perf report.
Use sample raw to mark memory mapping in aux buffer.
Signed-off-by: Athira Rajeev <atrajeev@linux.ibm.com>
---
arch/powerpc/perf/htm-perf.c | 110 ++++++++++++++++++++++++++++++++++-
1 file changed, 108 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/perf/htm-perf.c b/arch/powerpc/perf/htm-perf.c
index ae7f469b6840..fe458bc3ec05 100644
--- a/arch/powerpc/perf/htm-perf.c
+++ b/arch/powerpc/perf/htm-perf.c
@@ -76,6 +76,10 @@ struct htm_pmu_buf {
bool full;
int htm_stopped;
int collect_htm_trace;
+ u64 mem_head;
+ void *htm_mem_buf;
+ u64 mem_start;
+ int collect_htm_mem;
};
struct htm_pmu_ctx {
@@ -143,6 +147,86 @@ static ssize_t htm_return_check(int rc)
return -EINVAL;
}
+static int htm_collect_memory_config(struct perf_event *event,
+ struct htm_pmu_buf *aux_buf)
+{
+ struct perf_sample_data data;
+ struct perf_raw_record raw;
+ struct pt_regs regs;
+ u64 *num_entries;
+ u64 to_copy = 0;
+ int htm_val;
+ long rc;
+ int ret;
+ int retries = 0;
+ size_t size;
+ size_t space_to_end = aux_buf->size - aux_buf->mem_head;
+
+ /* Capture HTM system memory configuration in aux buffer */
+ do {
+ rc = htm_hcall_wrapper(htmflags, 0, 0, 0,
+ 0, H_HTM_OP_DUMP_SYSMEM_CONF, virt_to_phys(aux_buf->htm_mem_buf),
+ PAGE_SIZE, aux_buf->mem_start);
+ ret = htm_return_check(rc);
+ } while (ret == -EBUSY && ++retries < 100);
+
+ /* Return once there is no more data in HTM buffer */
+ if (ret <= 0) {
+ perf_sample_data_init(&data, 0, event->hw.last_period);
+ memset(&raw, 0, sizeof(raw));
+ memset(®s, 0, sizeof(regs));
+
+ htm_val = (aux_buf->head/((aux_buf->nr_pages * PAGE_SIZE)));
+ raw.frag.data = &htm_val;
+ raw.frag.size = sizeof(htm_val);
+
+ aux_buf->collect_htm_mem = 0;
+ perf_sample_save_raw_data(&data, event, &raw);
+ perf_event_overflow(event, &data, ®s);
+ return 0;
+ }
+
+ /*
+ * Find how much data to copy to aux buffer
+ * If hcall returned H_PARTIAL, set mem_start to
+ * indicate next offset of memory to read from
+ */
+ num_entries = aux_buf->htm_mem_buf + 0x10;
+ aux_buf->mem_start = be64_to_cpu(*(u64 *)(aux_buf->htm_mem_buf + 0x8));
+
+ to_copy = 32 + (be64_to_cpu(*num_entries) * 32);
+
+ if (to_copy <= space_to_end) {
+ if ((to_copy + aux_buf->mem_head) >= ((aux_buf->nr_pages * PAGE_SIZE)/2)) {
+ /*
+ * Crossing 50% threshold - flush and wrap.
+ * Write current chunk, then pad to end of buffer.
+ * This ensures next write starts at beginning with
+ * perf head also at beginning (synchronized).
+ */
+ memcpy(aux_buf->base + aux_buf->mem_head, aux_buf->htm_mem_buf, to_copy);
+ aux_buf->mem_head = 0;
+
+ /*
+ * Return space_to_end to include padding.
+ * Perf will advance head to end (wrapping to 0),
+ * matching our mem_head position.
+ */
+ size = space_to_end;
+ } else {
+ /* Normal case - chunk fits without crossing threshold */
+ memcpy(aux_buf->base + aux_buf->mem_head, aux_buf->htm_mem_buf, to_copy);
+ aux_buf->mem_head += to_copy;
+ size = to_copy;
+ }
+ } else {
+ return 0;
+ }
+
+ /* Return non-zero to indicate that one record is written to aux buffer */
+ return size;
+}
+
static int htm_dump_sample_data(struct perf_event *event)
{
struct htm_pmu_ctx *htm_ctx = this_cpu_ptr(&htm_pmu_ctx);
@@ -162,7 +246,7 @@ static int htm_dump_sample_data(struct perf_event *event)
if (!aux_buf)
return -1;
- if (!aux_buf->collect_htm_trace) {
+ if (!aux_buf->collect_htm_mem && !aux_buf->collect_htm_trace) {
perf_aux_output_end(&htm_ctx->handle, 0);
return 0;
}
@@ -202,12 +286,17 @@ static int htm_dump_sample_data(struct perf_event *event)
if (ret > 0) {
aux_buf->head += (aux_buf->nr_pages * PAGE_SIZE);
perf_aux_output_end(&htm_ctx->handle, (aux_buf->nr_pages * PAGE_SIZE));
+ return ret;
} else {
aux_buf->collect_htm_trace = 0;
- perf_aux_output_end(&htm_ctx->handle, 0);
}
}
+ if (aux_buf->collect_htm_mem) {
+ ret = htm_collect_memory_config(event, aux_buf);
+ perf_aux_output_end(&htm_ctx->handle, ret);
+ }
+
return ret;
}
@@ -397,6 +486,13 @@ static void *htm_setup_aux(struct perf_event *event, void **pages,
return NULL;
}
+ buf->htm_mem_buf = kmalloc_node(PAGE_SIZE, GFP_KERNEL, cpu_to_node(cpu));
+ if (!buf->htm_mem_buf) {
+ kfree(buf);
+ pr_err("Failed to allocate htm mem buf\n");
+ return NULL;
+ }
+
buf->nr_pages = nr_pages;
buf->snapshot = false;
buf->size = nr_pages << PAGE_SHIFT;
@@ -404,6 +500,9 @@ static void *htm_setup_aux(struct perf_event *event, void **pages,
buf->head_size = 0;
buf->htm_stopped = 0;
buf->collect_htm_trace = 1;
+ buf->mem_head = 0;
+ buf->collect_htm_mem = 1;
+ buf->mem_start = 0;
return buf;
}
@@ -413,10 +512,17 @@ static void *htm_setup_aux(struct perf_event *event, void **pages,
static void htm_free_aux(void *aux)
{
struct htm_pmu_buf *buf = aux;
+ void *free_mem;
if (!buf)
return;
+ free_mem = buf->htm_mem_buf;
+ buf->htm_mem_buf = NULL;
+
+ smp_mb();
+
+ kfree(free_mem);
kfree(buf);
}
--
2.52.0
^ permalink raw reply related
* [PATCH 2/5] powerpc/htm: Add support to setup and free aux buffer for capturing HTM data
From: Athira Rajeev @ 2026-07-01 8:38 UTC (permalink / raw)
To: linuxppc-dev, maddy
Cc: linux-perf-users, atrajeev, hbathini, tejas05, venkat88, tshah
In-Reply-To: <20260701083806.79358-1-atrajeev@linux.ibm.com>
HTM trace data is saved to perf.data when monitoring completes.
We directly copy the trace data as part of auxiliary buffer and it
will be postprocessed later. To enable the support for aux buffer,
add the PMU callbacks for setup_aux and free_aux.
In setup_aux, set up pmu-private data structures for an AUX
area. rb_alloc_aux uses "alloc_pages_node" and returns pointer to each
page address. "struct htm_pmu_buf" mainly saves:
1. buf->base: aux buffer base address
2. buf->head: offset from base address where data will be written to.
3. buf->size: Size of allocated memory
free_aux will free pmu-private AUX data structures.
Signed-off-by: Athira Rajeev <atrajeev@linux.ibm.com>
---
arch/powerpc/perf/htm-perf.c | 162 ++++++++++++++++++++++++++++++++++-
1 file changed, 160 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/perf/htm-perf.c b/arch/powerpc/perf/htm-perf.c
index e22a7fdce2f5..ae7f469b6840 100644
--- a/arch/powerpc/perf/htm-perf.c
+++ b/arch/powerpc/perf/htm-perf.c
@@ -66,6 +66,23 @@ static const struct attribute_group *attr_groups[] = {
static u64 htmflags = H_HTM_FLAGS_NOWRAP;
+struct htm_pmu_buf {
+ int nr_pages;
+ bool snapshot;
+ void *base;
+ u64 size;
+ u64 head;
+ u64 head_size;
+ bool full;
+ int htm_stopped;
+ int collect_htm_trace;
+};
+
+struct htm_pmu_ctx {
+ struct perf_output_handle handle;
+};
+
+static DEFINE_PER_CPU(struct htm_pmu_ctx, htm_pmu_ctx);
/*
* Check the return code for H_HTM hcall.
* Return non-zero value (1) if either H_PARTIAL or H_SUCCESS
@@ -126,6 +143,74 @@ static ssize_t htm_return_check(int rc)
return -EINVAL;
}
+static int htm_dump_sample_data(struct perf_event *event)
+{
+ struct htm_pmu_ctx *htm_ctx = this_cpu_ptr(&htm_pmu_ctx);
+ struct htm_pmu_buf *aux_buf;
+ u64 config = event->attr.config;
+ u32 htmtype, nodeindex, nodalchipindex, coreindexonchip;
+ long rc;
+ int ret = 0;
+ int retries = 0;
+
+ htmtype = config & 0xf;
+ nodeindex = (config >> 4) & 0xff;
+ nodalchipindex = (config >> 12) & 0xff;
+ coreindexonchip = (config >> 20) & 0xff;
+
+ aux_buf = perf_aux_output_begin(&htm_ctx->handle, event);
+ if (!aux_buf)
+ return -1;
+
+ if (!aux_buf->collect_htm_trace) {
+ perf_aux_output_end(&htm_ctx->handle, 0);
+ return 0;
+ }
+
+ if (!aux_buf->htm_stopped) {
+ do {
+ rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
+ htmtype, H_HTM_OP_STOP, 0, 0, 0);
+ ret = htm_return_check(rc);
+ } while (ret == -EBUSY && ++retries < 100);
+
+ if (ret > 0) {
+ /* HTM stopped trace collection */
+ aux_buf->htm_stopped = 1;
+ } else {
+ /* Failed to stop tracing, don't proceed to trace collection */
+ perf_aux_output_end(&htm_ctx->handle, 0);
+ return ret;
+ }
+ /* Reset the retries */
+ retries = 0;
+ }
+
+ /*
+ * Invoke H_HTM call with:
+ * - operation as htm dump (H_HTM_OP_DUMP_DATA)
+ * - last three values are address, size and offset
+ */
+ if (aux_buf->collect_htm_trace) {
+ do {
+ rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
+ htmtype, H_HTM_OP_DUMP_DATA, virt_to_phys(aux_buf->base),
+ (aux_buf->nr_pages * PAGE_SIZE), aux_buf->head);
+ ret = htm_return_check(rc);
+ } while (ret == -EBUSY && ++retries < 100);
+
+ if (ret > 0) {
+ aux_buf->head += (aux_buf->nr_pages * PAGE_SIZE);
+ perf_aux_output_end(&htm_ctx->handle, (aux_buf->nr_pages * PAGE_SIZE));
+ } else {
+ aux_buf->collect_htm_trace = 0;
+ perf_aux_output_end(&htm_ctx->handle, 0);
+ }
+ }
+
+ return ret;
+}
+
static int htm_event_init(struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
@@ -262,7 +347,77 @@ static void htm_event_del(struct perf_event *event, int flags)
*/
static void htm_event_read(struct perf_event *event)
{
- return;
+ int ret;
+
+ if (event->state != PERF_EVENT_STATE_ACTIVE)
+ return;
+
+ ret = htm_dump_sample_data(event);
+
+ if (ret <= 0)
+ local64_set(&event->count, 0);
+ else
+ local64_set(&event->count, 1);
+}
+
+/*
+ * Set up pmu-private data structures for an AUX area
+ * **pages contains the aux buffer allocated for this event
+ * for the corresponding cpu. rb_alloc_aux uses "alloc_pages_node"
+ * and returns pointer to each page address. Map these pages to
+ * contiguous space using vmap and use that as base address.
+ *
+ * The aux private data structure ie, "struct htm_pmu_buf" mainly
+ * saves
+ * - buf->base: aux buffer base address
+ * - buf->head: offset from base address where data will be written to.
+ * - buf->size: Size of allocated memory
+ */
+static void *htm_setup_aux(struct perf_event *event, void **pages,
+ int nr_pages, bool snapshot)
+{
+ int cpu = event->cpu;
+ struct htm_pmu_buf *buf;
+
+ /* We need at least one page for this to work. */
+ if (!nr_pages)
+ return NULL;
+
+ if (cpu == -1)
+ cpu = raw_smp_processor_id();
+
+ buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu));
+ if (!buf)
+ return NULL;
+
+ buf->base = pages[0];
+
+ if (!buf->base) {
+ kfree(buf);
+ return NULL;
+ }
+
+ buf->nr_pages = nr_pages;
+ buf->snapshot = false;
+ buf->size = nr_pages << PAGE_SHIFT;
+ buf->head = 0;
+ buf->head_size = 0;
+ buf->htm_stopped = 0;
+ buf->collect_htm_trace = 1;
+ return buf;
+}
+
+/*
+ * free pmu-private AUX data structures
+ */
+static void htm_free_aux(void *aux)
+{
+ struct htm_pmu_buf *buf = aux;
+
+ if (!buf)
+ return;
+
+ kfree(buf);
}
static void htm_event_start(struct perf_event *event, int flags)
@@ -284,7 +439,10 @@ static struct pmu htm_pmu = {
.read = htm_event_read,
.start = htm_event_start,
.stop = htm_event_stop,
- .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_EXCLUSIVE,
+ .setup_aux = htm_setup_aux,
+ .free_aux = htm_free_aux,
+ .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_EXCLUSIVE
+ | PERF_PMU_CAP_AUX_NO_SG | PERF_PMU_CAP_AUX_PREFER_LARGE,
};
static int htm_init(void)
--
2.52.0
^ permalink raw reply related
* [PATCH 1/5] powerpc/htm: Add interface to expose HTM trace data via perf
From: Athira Rajeev @ 2026-07-01 8:38 UTC (permalink / raw)
To: linuxppc-dev, maddy
Cc: linux-perf-users, atrajeev, hbathini, tejas05, venkat88, tshah
In-Reply-To: <20260701083806.79358-1-atrajeev@linux.ibm.com>
H_HTM (Hardware Trace Macro) hypervisor call is an HCALL to export data
from Hardware Trace Macro (HTM) function. Add support for setup,
configuration and control of HTM function via PMU.
H_HTM is used as an interface for executing Hardware Trace Macro (HTM)
functions, including setup, configuration, control and dumping of the
HTM data. HTM operations can be controlled using the H_HTM hcall. The
hcall can be invoked for any core/chip of the system from within a
partition itself.
To use this, expose event as part of "htm" PMU. The event code or config
is 28 bit value, where user can specify below required fields:
event: "config:0-27"
htm_type: "config:0-3"
nodeindex: "config:4-11"
nodalchipindex: "config:12-19"
coreindexonchip: "config:20-27"
1) nodeindex, nodalchipindex, coreindexonchip: this specifies
which partition to configure the HTM for.
2) htmtype: specifies the type of HTM.
In htm_event_add: configure and start the tracing using htm_hcall_wrapper
which is defined in plpar_wrappers.h header file
In htm_event_del: stop and deconfigure the tracing using
htm_hcall_wrapper
With the changes:
# ls /sys/bus/event_source/devices/ |grep htm
htm
# ls /sys/bus/event_source/devices/htm/
events format perf_event_mux_interval_ms power subsystem type uevent
Signed-off-by: Athira Rajeev <atrajeev@linux.ibm.com>
---
arch/powerpc/perf/Makefile | 2 +-
arch/powerpc/perf/htm-perf.c | 307 +++++++++++++++++++++++++++++++++++
2 files changed, 308 insertions(+), 1 deletion(-)
create mode 100644 arch/powerpc/perf/htm-perf.c
diff --git a/arch/powerpc/perf/Makefile b/arch/powerpc/perf/Makefile
index 78dd7e25219e..26ef30c0693c 100644
--- a/arch/powerpc/perf/Makefile
+++ b/arch/powerpc/perf/Makefile
@@ -14,7 +14,7 @@ obj-$(CONFIG_PPC_POWERNV) += imc-pmu.o
obj-$(CONFIG_FSL_EMB_PERF_EVENT) += core-fsl-emb.o
obj-$(CONFIG_FSL_EMB_PERF_EVENT_E500) += e500-pmu.o e6500-pmu.o
-obj-$(CONFIG_HV_PERF_CTRS) += hv-24x7.o hv-gpci.o hv-common.o vpa-dtl.o
+obj-$(CONFIG_HV_PERF_CTRS) += hv-24x7.o hv-gpci.o hv-common.o vpa-dtl.o htm-perf.o
obj-$(CONFIG_VPA_PMU) += vpa-pmu.o
diff --git a/arch/powerpc/perf/htm-perf.c b/arch/powerpc/perf/htm-perf.c
new file mode 100644
index 000000000000..e22a7fdce2f5
--- /dev/null
+++ b/arch/powerpc/perf/htm-perf.c
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Perf interface to expose HTM Trace data.
+ *
+ * Copyright (C) 2025 Athira Rajeev, IBM Corporation
+ */
+
+#define pr_fmt(fmt) "htm: " fmt
+
+#include <asm/dtl.h>
+#include <linux/perf_event.h>
+#include <asm/plpar_wrappers.h>
+#include <linux/vmalloc.h>
+
+extern void perf_event_wakeup(struct perf_event *event);
+#define EVENT(_name, _code) enum{_name = _code}
+
+/*
+ * H_HTM (Hardware Trace Macro) hypervisor call is an HCALL to export
+ * data from Hardware Trace Macro (HTM) function.
+ *
+ * Event codes based on HTM type.
+ */
+EVENT(HTM_CORE, 0x2);
+EVENT(HTM_NEST, 0x1);
+
+GENERIC_EVENT_ATTR(htm_core, HTM_CORE);
+GENERIC_EVENT_ATTR(htm_nest, HTM_NEST);
+
+PMU_FORMAT_ATTR(event, "config:0-27");
+PMU_FORMAT_ATTR(htm_type, "config:0-3");
+PMU_FORMAT_ATTR(nodeindex, "config:4-11");
+PMU_FORMAT_ATTR(nodalchipindex, "config:12-19");
+PMU_FORMAT_ATTR(coreindexonchip, "config:20-27");
+
+static struct attribute *events_attr[] = {
+ GENERIC_EVENT_PTR(HTM_NEST),
+ GENERIC_EVENT_PTR(HTM_CORE),
+ NULL
+};
+
+static struct attribute_group event_group = {
+ .name = "events",
+ .attrs = events_attr,
+};
+
+static struct attribute *format_attrs[] = {
+ &format_attr_event.attr,
+ &format_attr_htm_type.attr,
+ &format_attr_nodeindex.attr,
+ &format_attr_nodalchipindex.attr,
+ &format_attr_coreindexonchip.attr,
+ NULL,
+};
+
+static const struct attribute_group format_group = {
+ .name = "format",
+ .attrs = format_attrs,
+};
+
+static const struct attribute_group *attr_groups[] = {
+ &format_group,
+ &event_group,
+ NULL,
+};
+
+static u64 htmflags = H_HTM_FLAGS_NOWRAP;
+
+/*
+ * Check the return code for H_HTM hcall.
+ * Return non-zero value (1) if either H_PARTIAL or H_SUCCESS
+ * is returned. For other return codes:
+ * Return zero if H_NOT_AVAILABLE.
+ * Return -EBUSY if hcall return busy.
+ * Return -EINVAL if any parameter or operation is not valid.
+ * Return -EPERM if HTM Virtualization Engine Technology code
+ * is not applied.
+ * Return -EIO if the HTM state is not valid.
+ */
+static ssize_t htm_return_check(int rc)
+{
+ switch (rc) {
+ case H_SUCCESS:
+ break;
+ /* H_PARTIAL for the case where all available data can't be
+ * returned due to buffer size constraint.
+ */
+ case H_PARTIAL:
+ break;
+ /* H_NOT_AVAILABLE indicates reading from an offset outside the range,
+ * i.e. past end of file.
+ */
+ case H_NOT_AVAILABLE:
+ return 0;
+ case H_BUSY:
+ case H_LONG_BUSY_ORDER_1_MSEC:
+ case H_LONG_BUSY_ORDER_10_MSEC:
+ case H_LONG_BUSY_ORDER_100_MSEC:
+ case H_LONG_BUSY_ORDER_1_SEC:
+ case H_LONG_BUSY_ORDER_10_SEC:
+ case H_LONG_BUSY_ORDER_100_SEC:
+ return -EBUSY;
+ case H_PARAMETER:
+ goto out;
+ case H_P2:
+ goto out;
+ case H_P3:
+ goto out;
+ case H_P4:
+ goto out;
+ case H_P5:
+ goto out;
+ case H_P6:
+ return -EINVAL;
+ case H_STATE:
+ return -EIO;
+ case H_AUTHORITY:
+ return -EPERM;
+ }
+
+ /*
+ * Return 1 for H_SUCCESS/H_PARTIAL
+ */
+ return 1;
+out:
+ return -EINVAL;
+}
+
+static int htm_event_init(struct perf_event *event)
+{
+ struct hw_perf_event *hwc = &event->hw;
+ u64 config = event->attr.config;
+ u32 htmtype;
+
+ if (event->attr.inherit)
+ return -EOPNOTSUPP;
+
+ /* test the event attr type for PMU enumeration */
+ if (event->attr.type != event->pmu->type)
+ return -ENOENT;
+
+ if (!perfmon_capable())
+ return -EACCES;
+
+ /* Return if this is a counting event */
+ if (!is_sampling_event(event))
+ return -EOPNOTSUPP;
+
+ /* no branch sampling */
+ if (has_branch_stack(event))
+ return -EOPNOTSUPP;
+
+ htmtype = config & 0xf;
+ /* Invalid eventcode */
+ switch (htmtype) {
+ case HTM_CORE:
+ case HTM_NEST:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ htmflags = H_HTM_FLAGS_NOWRAP;
+
+ if (event->attr.freq) {
+ hwc->sample_period = event->attr.sample_period;
+ local64_set(&hwc->period_left, hwc->sample_period);
+ hwc->last_period = hwc->sample_period;
+ event->attr.freq = 0;
+ }
+
+ return 0;
+}
+
+static int htm_event_add(struct perf_event *event, int flags)
+{
+ int rc, ret;
+ unsigned long param1 = -1, param2 = -1;
+ int retries = 0;
+ u64 config = event->attr.config;
+ u32 htmtype, nodeindex, nodalchipindex, coreindexonchip;
+
+ /*
+ * Invoke H_HTM call with:
+ * operation as htm configure (H_HTM_OP_CONFIGURE)
+ * last three values are unused, hence set to zero
+ */
+ htmtype = config & 0xf;
+ nodeindex = (config >> 4) & 0xff;
+ nodalchipindex = (config >> 12) & 0xff;
+ coreindexonchip = (config >> 20) & 0xff;
+ do {
+ rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
+ htmtype, H_HTM_OP_CONFIGURE, param1, param2, 0);
+ ret = htm_return_check(rc);
+ } while (ret <= 0 && ++retries < 100);
+ if (ret <= 0)
+ return -1;
+
+ /* Reset retries */
+ retries = 0;
+
+ /*
+ * Invoke H_HTM call with:
+ * operation as htm start (H_HTM_OP_START)
+ * last three values are unused, hence set to zero
+ */
+ do {
+ rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
+ htmtype, H_HTM_OP_START, 0, 0, 0);
+ ret = htm_return_check(rc);
+ } while (ret == -EBUSY && ++retries < 100);
+
+ if (htm_return_check(rc) <= 0)
+ return -1;
+
+ return 0;
+}
+
+static void htm_event_del(struct perf_event *event, int flags)
+{
+ long rc;
+ int ret;
+ int retries = 0;
+ u64 config = event->attr.config;
+ u32 htmtype, nodeindex, nodalchipindex, coreindexonchip;
+
+ /*
+ * Invoke H_HTM call with:
+ * operation as htm stop (H_HTM_OP_STOP)
+ * last three values are unused, hence set to zero
+ */
+ htmtype = config & 0xf;
+ nodeindex = (config >> 4) & 0xff;
+ nodalchipindex = (config >> 12) & 0xff;
+ coreindexonchip = (config >> 20) & 0xff;
+ do {
+ rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
+ htmtype, H_HTM_OP_STOP, 0, 0, 0);
+ ret = htm_return_check(rc);
+ } while (ret == -EBUSY && ++retries < 100);
+
+ /* Reset retries */
+ retries = 0;
+
+ /*
+ * Invoke H_HTM call with:
+ * operation as htm configure (H_HTM_OP_DECONFIGURE)
+ * last three values are unused, hence set to zero
+ */
+ do {
+ rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip,
+ htmtype, H_HTM_OP_DECONFIGURE, 0, 0, 0);
+ ret = htm_return_check(rc);
+ } while (ret <= 0 && ++retries < 100);
+}
+
+/*
+ * This function definition is empty as htm_dump_sample_data
+ * is used to parse and dump the HTM trace data,
+ * to perf data.
+ */
+static void htm_event_read(struct perf_event *event)
+{
+ return;
+}
+
+static void htm_event_start(struct perf_event *event, int flags)
+{
+}
+
+static void htm_event_stop(struct perf_event *event, int flags)
+{
+}
+
+static struct pmu htm_pmu = {
+ .task_ctx_nr = perf_invalid_context,
+
+ .name = "htm",
+ .attr_groups = attr_groups,
+ .event_init = htm_event_init,
+ .add = htm_event_add,
+ .del = htm_event_del,
+ .read = htm_event_read,
+ .start = htm_event_start,
+ .stop = htm_event_stop,
+ .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_EXCLUSIVE,
+};
+
+static int htm_init(void)
+{
+ int r;
+
+ /* This driver is intended only for L1 host. */
+ if (is_kvm_guest()) {
+ pr_debug("Only supported for L1 host system\n");
+ return -ENODEV;
+ }
+
+ r = perf_pmu_register(&htm_pmu, htm_pmu.name, -1);
+ if (r)
+ return r;
+
+ return 0;
+}
+
+device_initcall(htm_init);
--
2.52.0
^ permalink raw reply related
* [PATCH 0/5] powerpc/htm: Add interface to expose HTM trace data via perf
From: Athira Rajeev @ 2026-07-01 8:38 UTC (permalink / raw)
To: linuxppc-dev, maddy
Cc: linux-perf-users, atrajeev, hbathini, tejas05, venkat88, tshah
H_HTM (Hardware Trace Macro) hypervisor call is an HCALL to export data
from Hardware Trace Macro (HTM) function. Patchset adds support for setup,
configuration and control of HTM functions as well as trace data
collection via perf PMU interface.
H_HTM is used as an interface for executing Hardware Trace Macro (HTM)
functions, including setup, configuration, control and dumping of the
HTM trace data. HTM operations can be controlled using the H_HTM hcall.
The hcall can be invoked for any core/chip of the system from within a
partition itself.
HTM perf interface usage:
The HTM (Hardware Trace Macro) perf interface enables collection and
analysis of hardware trace data from PowerPC systems. This interface
allows users to capture detailed execution traces for performance
analysis and debugging. The interface uses AUX infrastructure for
capturing of trace data.
Patchset includes powerpc kernel side changes.
Patch 1 introduces the HTM PMU interface with event configuration
support for specifying target hardware (node, chip, core) and trace
type. It integrates H_HTM hcall wrappers for starting, stopping, and
controlling HTM trace collection.
Patch 2 implements AUX buffer infrastructure by adding setup_aux and
free_aux callbacks, enabling perf to allocate and manage auxiliary
buffers for HTM data.
Patch 3 extends trace collection to capture system memory
configuration alongside trace data, using PERF_SAMPLE_RAW records to
mark data boundaries in the AUX buffer for post-processing.
Patches 4 and 5 provide comprehensive documentation for the HTM PMU
interface, including ABI documentation for sysfs attributes and
user-facing documentation with usage examples and workflows.
perf tools side patches will be posted separately in the linux-perf-users
mailing list.
Event Configuration:
Use "perf record" with the htm PMU event. The event is configured using
named parameters that specify the target hardware location and trace type:
- htm_type
- Type of HTM trace to collect (bits 0-3)
- nodeindex
- Node index in the system topology (bits 4-11)
- nodalchipindex
- Chip index within the specified node (bits 12-19)
- coreindexonchip
- Core index on the specified chip (bits 20-27)
event: "config:0-27"
htm_type: "config:0-3"
nodeindex: "config:4-11"
nodalchipindex: "config:12-19"
coreindexonchip: "config:20-27"
1) nodeindex, nodalchipindex, coreindexonchip: this specifies
which partition to configure the HTM for.
2) htmtype: specifies the type of HTM.
Event Syntax:
The event configuration uses named parameters::
htm/nodeindex=N,nodalchipindex=C,coreindexonchip=R,htm_type=T/
Where:
- N = node index
- C = chip index within the node
- R = core index on the chip
- T = HTM type
Basic Usage Example:
To collect HTM trace data for a specific chip:
# perf record -C 1 -e htm/nodalchipindex=2,nodeindex=0,htm_type=1/ <workload>
In this example:
- nodeindex=0: Target node 0
- nodalchipindex=2: Target chip 2 within node 0
- htm_type=1: HTM trace type 1
Output Files:
After running "perf record", the following files are generated:
# ls htm.bin.*
htm.bin.n0.p2.c0 htm.bin.n1.p3.c0 # Binary trace files
# ls translation.*
translation.n0.p2.c0 translation.n1.p3.c0 # Memory configuration files
These files contain:
- **htm.bin.*** - Raw HTM trace data in binary format
- **translation.*** - Memory address translation information for decoding
Trace Data Processing:
Process the collected trace data using perf script:
# perf script -D
This command:
1. Reads the perf.data file
2. Decodes HTM trace data using translation files
3. Displays human-readable trace output
The decoder automatically:
- Translates physical addresses to logical addresses
- Creates decoded output files for analysis
- Correlates trace data with memory mappings
Here's a complete example of collecting and analyzing HTM traces:
# Step 1: Collect trace data
perf record -C 1 -e htm/nodalchipindex=2,nodeindex=0,htm_type=1/ sleep 5
# Step 2: Verify output files
ls htm.bin.* # Binary trace files
ls translation.* # Memory configuration files
ls perf.data # Perf data file
# Step 3: Decode and view traces
perf script -D > decoded_trace.txt
# Step 4: Analyze with perf report to see the hot logical address
perf report
Thanks
Athira
Athira Rajeev (5):
powerpc/htm: Add interface to expose HTM trace data via perf
powerpc/htm: Add support to setup and free aux buffer for capturing
HTM data
powerpc/perf: Capture the HTM memory configuration as part of perf
data
docs: ABI: sysfs-bus-event_source-devices-htm: Document sysfs event
format entries for htm pmu
powerpc/perf/htm: Add documentation for Hardware Trace Macro PMU
.../sysfs-bus-event_source-devices-htm | 21 +
Documentation/arch/powerpc/htm.rst | 137 ++++-
arch/powerpc/perf/Makefile | 2 +-
arch/powerpc/perf/htm-perf.c | 571 ++++++++++++++++++
4 files changed, 727 insertions(+), 4 deletions(-)
create mode 100644 Documentation/ABI/testing/sysfs-bus-event_source-devices-htm
create mode 100644 arch/powerpc/perf/htm-perf.c
--
2.52.0
^ permalink raw reply
* Re: [PATCH V2] powerpc/syscall: Fix seccomp errno handling with GENERIC_ENTRY
From: Michal Suchánek @ 2026-07-01 8:29 UTC (permalink / raw)
To: Mukesh Kumar Chaurasiya
Cc: Shrikanth Hegde, maddy, mpe, npiggin, chleroy, mkchauras,
ryan.roberts, ruanjinjie, linuxppc-dev, linux-kernel
In-Reply-To: <akTJbfXPKc10Xemh@kunlun.suse.cz>
On Wed, Jul 01, 2026 at 10:01:49AM +0200, Michal Suchánek wrote:
> On Wed, Jul 01, 2026 at 09:41:57AM +0200, Michal Suchánek wrote:
> > On Wed, Jul 01, 2026 at 11:57:00AM +0530, Mukesh Kumar Chaurasiya wrote:
> > > On Wed, Jul 01, 2026 at 01:41:09AM +0530, Shrikanth Hegde wrote:
> > > > Hi Mukesh.
> > > >
> > > > On 6/29/26 11:59 PM, Mukesh Kumar Chaurasiya (IBM) wrote:
> > > > > diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c
> > > > > index a9da2af6efa8..36d73933a311 100644
> > > > > --- a/arch/powerpc/kernel/syscall.c
> > > > > +++ b/arch/powerpc/kernel/syscall.c
> > > > > @@ -20,7 +20,6 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
> > > > > syscall_fn f;
> > > > > add_random_kstack_offset();
> > > > > - r0 = syscall_enter_from_user_mode(regs, r0);
> > > > > if (unlikely(r0 >= NR_syscalls)) {
> > > > > if (unlikely(trap_is_unsupported_scv(regs))) {
> > > > > @@ -31,6 +30,12 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
> > > > > return -ENOSYS;
> > > > > }
> > > > > + r0 = syscall_enter_from_user_mode(regs, r0);
> > > > > +
> > > >
> > > > I see many arch first do syscall_enter_from_user_mode and then check for return value.
> > > > take x86 for example,
> > > >
> > > > __visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr)
> > > > {
> > > > nr = syscall_enter_from_user_mode(regs, nr);
> > > >
> > > > if (!do_syscall_x64(regs, nr) && !do_syscall_x32(regs, nr) && nr != -1) {
> > > > /* Invalid system call, but still a system call. */
> > > > regs->ax = __x64_sys_ni_syscall(regs);
> > > > }
> > > >
> > > > }
> > > >
> > > > So seccomp fails silently there if initial nr was -1?
> > > >
> > > Hey,
> > >
> > > No the -1 syscall ignores the error silently and returns 0.
> > >
> >
> > There seems to be some inconsistency with the invalid syscalls.
> >
> > Adapting the example from seccomp man page to ignore architecture I get
> > on x86_64 (presumably with GENERIC_ENTRY since long ago):
> >
> > ./a.out -2 55 /usr/bin/perl -MPOSIX -e '$!=0; my $r = syscall(-2, 0); print "ret=$r errno=".($!+0)." ($!)\n"'
> > ret=-1 errno=55 (No anode)
> >
> > but on ppc64le (with GENEREC_ENTRY):
> >
> > ./a.out -2 55 /usr/bin/perl -MPOSIX -e '$!=0; my $r = syscall(-2, 0); print "ret=$r errno=".($!+0)." ($!)\n"'
> > ret=-1 errno=38 (Function not implemented)
> >
> > That said, behavior of seccomp on invalid syscalls is not particularly
> > concerning. The tools that people typically use for constructing those
> > filters typically require a valid syscall number.
> >
> > It would be nice to align, though.
>
> It is more concerning for SECCOMP_SET_MODE_STRICT or similar. So it
> should be resolved to correctly execute seccomp even on invalid
> syscalls. The syscall_enter_from_user_mode API is not particularly
> well-suited for that, though.
In particular the fixup per
https://lore.kernel.org/linuxppc-dev/akJzuEJRLniHk4Fi@kunlun.suse.cz/
handles some cases
./a.out -2 55 /usr/bin/perl -MPOSIX -e '$!=0; my $r = syscall(-2, 0); print "ret=$r errno=".($!+0)." ($!)\n"'
ret=-1 errno=55 (No anode)
but not -1
./a.out -1 55 /usr/bin/perl -MPOSIX -e '$!=0; my $r = syscall(-1, 0); print "ret=$r errno=".($!+0)." ($!)\n"'
ret=-1 errno=38 (Function not implemented)
which is the direct result of the ambiguous return value of
syscall_enter_from_user_mode
Thanks
Michal
^ permalink raw reply
* Re: [PATCH v3] powerpc/audit: Convert powerpc to AUDIT_ARCH_COMPAT_GENERIC
From: Venkat Rao Bagalkote @ 2026-07-01 8:09 UTC (permalink / raw)
To: Christophe Leroy (CS GROUP), Madhavan Srinivasan, Paul Moore
Cc: Harsh Prateek Bora, Michael Ellerman, Nicholas Piggin, Eric Paris,
linux-kernel, linuxppc-dev, audit, Thomas Weissschuh,
Cédric Le Goater, ritesh.list
In-Reply-To: <b1d6e9a5-34ce-42ca-906a-d4237e18be25@kernel.org>
On 01/07/26 10:26 am, Christophe Leroy (CS GROUP) wrote:
>
>
> Le 01/07/2026 à 06:32, Venkat Rao Bagalkote a écrit :
>>
>> On 01/07/26 7:44 am, Madhavan Srinivasan wrote:
>>>
>>> On 7/1/26 12:41 AM, Paul Moore wrote:
>>>> On Wed, May 13, 2026 at 1:42 AM Madhavan Srinivasan
>>>> <maddy@linux.ibm.com> wrote:
>>>>> On 5/13/26 10:05 AM, Harsh Prateek Bora wrote:
>>>>>> On 11/03/26 12:49 am, Paul Moore wrote:
>>>>>>> On Tue, Mar 10, 2026 at 11:08 AM Christophe Leroy (CS GROUP)
>>>>>>> <chleroy@kernel.org> wrote:
>>>>>>>> From: Christophe Leroy <christophe.leroy@csgroup.eu>
>>>>>>>>
>>>>>>>> Commit e65e1fc2d24b ("[PATCH] syscall class hookup for all normal
>>>>>>>> targets") added generic support for AUDIT but that didn't include
>>>>>>>> support for bi-arch like powerpc.
>>>>>>>>
>>>>>>>> Commit 4b58841149dc ("audit: Add generic compat syscall support")
>>>>>>>> added generic support for bi-arch.
>>>>>>>>
>>>>>>>> Convert powerpc to that bi-arch generic audit support.
>>>>>>>>
>>>>>>>> With this change generated text is similar.
>>>>>>>>
>>>>>>>> Thomas has confirmed that the previously failing
>>>>>>>> filter_exclude/test
>>>>>>>> is now successful both without and with this patch, see [1]
>>>>>>>>
>>>>>>>> [1]
>>>>>>>> https://eur01.safelinks.protection.outlook.com/?
>>>>>>>> url=https%3A%2F%2Flore.kernel.org%2Fall%2F20260306115350-
>>>>>>>> ef265661-6d6b-4043-9bd0-8e6b437d0d67%40linutronix.de%2F&data=05%7C02%7Cchristophe.leroy%40csgroup.eu%7C81e1e4e3bae245103d3308ded729bb47%7C8b87af7d86474dc78df45f69a2011bb5%7C0%7C0%7C639184771399964577%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=%2FtCHXFR5np67gntCnXqv7Eemo5WuwaIEcywxZQzDADA%3D&reserved=0
>>>>>>>>
>>>>>>>>
>>>>>>>> Link: https://eur01.safelinks.protection.outlook.com/?
>>>>>>>> url=https%3A%2F%2Fgithub.com%2Flinuxppc%2Fissues%2Fissues%2F412&data=05%7C02%7Cchristophe.leroy%40csgroup.eu%7C81e1e4e3bae245103d3308ded729bb47%7C8b87af7d86474dc78df45f69a2011bb5%7C0%7C0%7C639184771400190794%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=4fPEX7HuGBcxOcXlcJVQvDdP7Y9InhDpbz3z%2BS9lkCQ%3D&reserved=0
>>>>>>>> Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
>>>>>>>> Reviewed-by: Cédric Le Goater <clg@kaod.org>
>>>>>>>> ---
>>>>>>>> Venkat, a test result with
>>>>>>>> https://eur01.safelinks.protection.outlook.com/?
>>>>>>>> url=https%3A%2F%2Fgithub.com%2Flinux-audit%2Faudit-
>>>>>>>> testsuite&data=05%7C02%7Cchristophe.leroy%40csgroup.eu%7C81e1e4e3bae245103d3308ded729bb47%7C8b87af7d86474dc78df45f69a2011bb5%7C0%7C0%7C639184771400214912%7CUnknown%7CTWFpbGZsb3d8eyJFbXB0eU1hcGkiOnRydWUsIlYiOiIwLjAuMDAwMCIsIlAiOiJXaW4zMiIsIkFOIjoiTWFpbCIsIldUIjoyfQ%3D%3D%7C0%7C%7C%7C&sdata=yHp3p5zNx%2BqQg2cKBsKJADWcUcKVtZstzNScBfWzF%2FQ%3D&reserved=0
>>>>>>>> would be appreciated.
>>>>>>> Yes, I'd like to see confirmation that the audit test suite runs
>>>>>>> clean
>>>>>>> on ppc systems with this patch applied, and unfortunately without a
>>>>>>> ppc system I have no way to test this myself.
>>>>> My bad, this is a miss from my end.
>>>>> Venkat is already on this and will update the results here.
>>>> Do we have an update on this? Maybe I missed it, but I don't recall
>>>> seeing any test results.
>>> Venkat, did run the test but I guess he missed to respond here,
>>> Details of his test run that he shared me in the internal chat from
>>> May 13.
>>
>>
>> Sorry, My bad. I missed updating here. Yes, this was tested in May.
>> Please let me know, if a re-run is required?
>
> It looks like netcat (https://linux.die.net/man/1/nc) is not installed
> in you setup. Are you able to install it and rerun the test ?
I investigated the failure on the RHEL 10 ppc64le test system.
The netfilter_pkt failure is due to missing userspace dependencies:
Can't exec "nc": No such file or directory
Can't exec "iptables": No such file or directory
Can't exec "ip6tables": No such file or directory
I checked the system and confirmed that nc, ncat, iptables, and
ip6tables are not installed:
which nc
which ncat
which iptables
which ip6tables
all return "not found".
The system has nftables installed (/usr/sbin/nft), but I was unable to
locate packages providing nc/ncat or iptables/ip6tables in the currently
enabled repositories (epel, rh10_base, rh10_app, rh10_crb).
At this point, the remaining failure appears to be an environmental
dependency issue rather than a test failure. Please let me know if there
is a recommended package/repository for EL10 ppc64le that provides
netcat and iptables compatibility tools, and I can re-run the test.
Regards,
Venkat.
>
> Thanks
> Christophe
>
>>
>> Regards,
>>
>> Venkat.
>>
>>>
>>> backlog_wait_time_actual_reset/test .. ok
>>> bpf/test ............................. ok
>>> exec_execve/test ..................... ok
>>> exec_name/test ....................... ok
>>> fanotify/test ........................ ok
>>> field_compare/test ................... ok
>>> file_create/test ..................... ok
>>> file_delete/test ..................... ok
>>> file_permission/test ................. ok
>>> file_rename/test ..................... ok
>>> filter_exclude/test .................. ok
>>> filter_exit/test ..................... ok
>>> filter_saddr_fam/test ................ ok
>>> filter_sessionid/test ................ ok
>>> io_uring/test ........................ ok
>>> login_tty/test ....................... ok
>>> lost_reset/test ...................... ok
>>> netfilter_pkt/test ................... Can't exec "nc": No such file
>>> or directory at netfilter_pkt/test line 83.
>>>
>>> Venkat, can you please re-run if possible and paste the log here.
>>>
>>> Thanks
>>> Maddy
>>>
>>>
>
^ permalink raw reply
* Re: [PATCH V2] powerpc/syscall: Fix seccomp errno handling with GENERIC_ENTRY
From: Michal Suchánek @ 2026-07-01 8:01 UTC (permalink / raw)
To: Mukesh Kumar Chaurasiya
Cc: Shrikanth Hegde, maddy, mpe, npiggin, chleroy, mkchauras,
ryan.roberts, ruanjinjie, linuxppc-dev, linux-kernel
In-Reply-To: <akTExSO3ZT7iRtBa@kunlun.suse.cz>
On Wed, Jul 01, 2026 at 09:41:57AM +0200, Michal Suchánek wrote:
> On Wed, Jul 01, 2026 at 11:57:00AM +0530, Mukesh Kumar Chaurasiya wrote:
> > On Wed, Jul 01, 2026 at 01:41:09AM +0530, Shrikanth Hegde wrote:
> > > Hi Mukesh.
> > >
> > > On 6/29/26 11:59 PM, Mukesh Kumar Chaurasiya (IBM) wrote:
> > > > After enabling GENERIC_ENTRY on PowerPC, seccomp filters using
> > > > SCMP_ACT_ERRNO without an explicit errnoRet value return ENOSYS
> > > > (Function not implemented) instead of the expected EPERM (Operation
> > > > not permitted).
> > > >
> > > > The issue occurs in system_call_exception() when syscall_enter_from_user_mode()
> > > > returns -1 to indicate the syscall should be skipped (e.g., blocked by seccomp).
> > > > The current code treats this -1 as a syscall number and compares it against
> > > > NR_syscalls. Since -1 is greater than NR_syscalls,
> > > > the code incorrectly returns -ENOSYS, overwriting the errno that seccomp
> > > > already set via syscall_set_return_value().
> > > >
> > > > The generic entry code in syscall_trace_enter() calls __secure_computing(),
> > > > which sets the appropriate errno in regs->gpr[3] and returns -1 to signal
> > > > that the syscall should be skipped. However, the PowerPC syscall handler
> > > > was not checking for this -1 return value before validating the syscall
> > > > number.
> > > >
> > > > Fix this by explicitly checking if syscall_enter_from_user_mode() returns
> > > > -1 and returning the value already set in regs->gpr[3] (the errno from
> > > > seccomp) before performing the syscall number validation.
> > > >
> > > > Also Move the syscall_enter_from_user_mode() call and the seccomp/ptrace
> > > > skip check to after the NR_syscalls bounds check.
> > > >
> > > > When syscall -1 was passed, the r0 == -1L check would trigger before
> > > > the NR_syscalls check, causing syscall_get_error() to return 0 instead
> > > > of -ENOSYS. This resulted in a silent success (ret=0, errno=0) instead
> > > > of the expected ENOSYS error.
> > > >
> > > > By moving syscall_enter_from_user_mode() after the bounds check, an
> > > > initial syscall number of -1 is correctly rejected with -ENOSYS first.
> > > > The seccomp/ptrace skip path still works correctly for valid syscall
> > > > numbers that get overridden to -1 by seccomp or ptrace.
> > > >
> > > > This aligns PowerPC's behavior with other architectures using GENERIC_ENTRY
> > > > and restores correct seccomp errno handling.
> > > >
> > > > Fixes: bee25f97ad24 ("powerpc: Enable GENERIC_ENTRY feature")
> > > > Reported-by: Michal Suchánek <msuchanek@suse.de>
> > > > Closes: https://lore.kernel.org/all/ajpp-_XnbF3UTM_E@kunlun.suse.cz/
> > > > Signed-off-by: Mukesh Kumar Chaurasiya (IBM) <mkchauras@gmail.com>
> > > > ---
> > > >
> > > > v1 -> v2:
> > > > - Fix issues in the previous fix (Michal)
> > > > v1: https://lore.kernel.org/all/20260624171520.772408-1-mkchauras@gmail.com
> > > >
> > > > arch/powerpc/kernel/syscall.c | 7 ++++++-
> > > > 1 file changed, 6 insertions(+), 1 deletion(-)
> > > >
> > > > diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c
> > > > index a9da2af6efa8..36d73933a311 100644
> > > > --- a/arch/powerpc/kernel/syscall.c
> > > > +++ b/arch/powerpc/kernel/syscall.c
> > > > @@ -20,7 +20,6 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
> > > > syscall_fn f;
> > > > add_random_kstack_offset();
> > > > - r0 = syscall_enter_from_user_mode(regs, r0);
> > > > if (unlikely(r0 >= NR_syscalls)) {
> > > > if (unlikely(trap_is_unsupported_scv(regs))) {
> > > > @@ -31,6 +30,12 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
> > > > return -ENOSYS;
> > > > }
> > > > + r0 = syscall_enter_from_user_mode(regs, r0);
> > > > +
> > >
> > > I see many arch first do syscall_enter_from_user_mode and then check for return value.
> > > take x86 for example,
> > >
> > > __visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr)
> > > {
> > > nr = syscall_enter_from_user_mode(regs, nr);
> > >
> > > if (!do_syscall_x64(regs, nr) && !do_syscall_x32(regs, nr) && nr != -1) {
> > > /* Invalid system call, but still a system call. */
> > > regs->ax = __x64_sys_ni_syscall(regs);
> > > }
> > >
> > > }
> > >
> > > So seccomp fails silently there if initial nr was -1?
> > >
> > Hey,
> >
> > No the -1 syscall ignores the error silently and returns 0.
> >
>
> There seems to be some inconsistency with the invalid syscalls.
>
> Adapting the example from seccomp man page to ignore architecture I get
> on x86_64 (presumably with GENERIC_ENTRY since long ago):
>
> ./a.out -2 55 /usr/bin/perl -MPOSIX -e '$!=0; my $r = syscall(-2, 0); print "ret=$r errno=".($!+0)." ($!)\n"'
> ret=-1 errno=55 (No anode)
>
> but on ppc64le (with GENEREC_ENTRY):
>
> ./a.out -2 55 /usr/bin/perl -MPOSIX -e '$!=0; my $r = syscall(-2, 0); print "ret=$r errno=".($!+0)." ($!)\n"'
> ret=-1 errno=38 (Function not implemented)
>
> That said, behavior of seccomp on invalid syscalls is not particularly
> concerning. The tools that people typically use for constructing those
> filters typically require a valid syscall number.
>
> It would be nice to align, though.
It is more concerning for SECCOMP_SET_MODE_STRICT or similar. So it
should be resolved to correctly execute seccomp even on invalid
syscalls. The syscall_enter_from_user_mode API is not particularly
well-suited for that, though.
Thanks
Michal
^ permalink raw reply
* Re: [PATCH V2] powerpc/syscall: Fix seccomp errno handling with GENERIC_ENTRY
From: Michal Suchánek @ 2026-07-01 7:41 UTC (permalink / raw)
To: Mukesh Kumar Chaurasiya
Cc: Shrikanth Hegde, maddy, mpe, npiggin, chleroy, mkchauras,
ryan.roberts, ruanjinjie, linuxppc-dev, linux-kernel
In-Reply-To: <akSyVOsraz9waBf4@li-1a3e774c-28e4-11b2-a85c-acc9f2883e29.ibm.com>
[-- Attachment #1: Type: text/plain, Size: 5028 bytes --]
On Wed, Jul 01, 2026 at 11:57:00AM +0530, Mukesh Kumar Chaurasiya wrote:
> On Wed, Jul 01, 2026 at 01:41:09AM +0530, Shrikanth Hegde wrote:
> > Hi Mukesh.
> >
> > On 6/29/26 11:59 PM, Mukesh Kumar Chaurasiya (IBM) wrote:
> > > After enabling GENERIC_ENTRY on PowerPC, seccomp filters using
> > > SCMP_ACT_ERRNO without an explicit errnoRet value return ENOSYS
> > > (Function not implemented) instead of the expected EPERM (Operation
> > > not permitted).
> > >
> > > The issue occurs in system_call_exception() when syscall_enter_from_user_mode()
> > > returns -1 to indicate the syscall should be skipped (e.g., blocked by seccomp).
> > > The current code treats this -1 as a syscall number and compares it against
> > > NR_syscalls. Since -1 is greater than NR_syscalls,
> > > the code incorrectly returns -ENOSYS, overwriting the errno that seccomp
> > > already set via syscall_set_return_value().
> > >
> > > The generic entry code in syscall_trace_enter() calls __secure_computing(),
> > > which sets the appropriate errno in regs->gpr[3] and returns -1 to signal
> > > that the syscall should be skipped. However, the PowerPC syscall handler
> > > was not checking for this -1 return value before validating the syscall
> > > number.
> > >
> > > Fix this by explicitly checking if syscall_enter_from_user_mode() returns
> > > -1 and returning the value already set in regs->gpr[3] (the errno from
> > > seccomp) before performing the syscall number validation.
> > >
> > > Also Move the syscall_enter_from_user_mode() call and the seccomp/ptrace
> > > skip check to after the NR_syscalls bounds check.
> > >
> > > When syscall -1 was passed, the r0 == -1L check would trigger before
> > > the NR_syscalls check, causing syscall_get_error() to return 0 instead
> > > of -ENOSYS. This resulted in a silent success (ret=0, errno=0) instead
> > > of the expected ENOSYS error.
> > >
> > > By moving syscall_enter_from_user_mode() after the bounds check, an
> > > initial syscall number of -1 is correctly rejected with -ENOSYS first.
> > > The seccomp/ptrace skip path still works correctly for valid syscall
> > > numbers that get overridden to -1 by seccomp or ptrace.
> > >
> > > This aligns PowerPC's behavior with other architectures using GENERIC_ENTRY
> > > and restores correct seccomp errno handling.
> > >
> > > Fixes: bee25f97ad24 ("powerpc: Enable GENERIC_ENTRY feature")
> > > Reported-by: Michal Suchánek <msuchanek@suse.de>
> > > Closes: https://lore.kernel.org/all/ajpp-_XnbF3UTM_E@kunlun.suse.cz/
> > > Signed-off-by: Mukesh Kumar Chaurasiya (IBM) <mkchauras@gmail.com>
> > > ---
> > >
> > > v1 -> v2:
> > > - Fix issues in the previous fix (Michal)
> > > v1: https://lore.kernel.org/all/20260624171520.772408-1-mkchauras@gmail.com
> > >
> > > arch/powerpc/kernel/syscall.c | 7 ++++++-
> > > 1 file changed, 6 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c
> > > index a9da2af6efa8..36d73933a311 100644
> > > --- a/arch/powerpc/kernel/syscall.c
> > > +++ b/arch/powerpc/kernel/syscall.c
> > > @@ -20,7 +20,6 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
> > > syscall_fn f;
> > > add_random_kstack_offset();
> > > - r0 = syscall_enter_from_user_mode(regs, r0);
> > > if (unlikely(r0 >= NR_syscalls)) {
> > > if (unlikely(trap_is_unsupported_scv(regs))) {
> > > @@ -31,6 +30,12 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
> > > return -ENOSYS;
> > > }
> > > + r0 = syscall_enter_from_user_mode(regs, r0);
> > > +
> >
> > I see many arch first do syscall_enter_from_user_mode and then check for return value.
> > take x86 for example,
> >
> > __visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr)
> > {
> > nr = syscall_enter_from_user_mode(regs, nr);
> >
> > if (!do_syscall_x64(regs, nr) && !do_syscall_x32(regs, nr) && nr != -1) {
> > /* Invalid system call, but still a system call. */
> > regs->ax = __x64_sys_ni_syscall(regs);
> > }
> >
> > }
> >
> > So seccomp fails silently there if initial nr was -1?
> >
> Hey,
>
> No the -1 syscall ignores the error silently and returns 0.
>
There seems to be some inconsistency with the invalid syscalls.
Adapting the example from seccomp man page to ignore architecture I get
on x86_64 (presumably with GENERIC_ENTRY since long ago):
./a.out -2 55 /usr/bin/perl -MPOSIX -e '$!=0; my $r = syscall(-2, 0); print "ret=$r errno=".($!+0)." ($!)\n"'
ret=-1 errno=55 (No anode)
but on ppc64le (with GENEREC_ENTRY):
./a.out -2 55 /usr/bin/perl -MPOSIX -e '$!=0; my $r = syscall(-2, 0); print "ret=$r errno=".($!+0)." ($!)\n"'
ret=-1 errno=38 (Function not implemented)
That said, behavior of seccomp on invalid syscalls is not particularly
concerning. The tools that people typically use for constructing those
filters typically require a valid syscall number.
It would be nice to align, though.
Thanks
Michal
[-- Attachment #2: seccomp.c --]
[-- Type: text/x-c, Size: 1757 bytes --]
#include <linux/audit.h>
#include <linux/filter.h>
#include <linux/seccomp.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/prctl.h>
#include <sys/syscall.h>
#include <unistd.h>
static int
install_filter(int syscall_nr, int f_errno)
{
struct sock_filter filter[] = {
/* [0] Load architecture from 'seccomp_data' buffer into
accumulator. */
BPF_STMT(BPF_LD | BPF_W | BPF_ABS,
(offsetof(struct seccomp_data, arch))),
/* [1] Load system call number from 'seccomp_data' buffer into
accumulator. */
BPF_STMT(BPF_LD | BPF_W | BPF_ABS,
(offsetof(struct seccomp_data, nr))),
/* [2] Jump forward 1 instruction if system call number
does not match 'syscall_nr'. */
BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, syscall_nr, 0, 1),
/* [3] Matching system call: don't execute
the system call, and return 'f_errno' in 'errno'. */
BPF_STMT(BPF_RET | BPF_K,
SECCOMP_RET_ERRNO | (f_errno & SECCOMP_RET_DATA)),
/* [4] Destination of system call number mismatch: allow other
system calls. */
BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW),
};
struct sock_fprog prog = {
.len = sizeof(filter) / sizeof(*filter),
.filter = filter,
};
if (syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &prog)) {
perror("seccomp");
return 1;
}
return 0;
}
int
main(int argc, char *argv[])
{
if (argc < 4) {
fprintf(stderr, "Usage: "
"%s <syscall_nr> <errno> <prog> [<args>]\n"
"\n", argv[0]);
exit(EXIT_FAILURE);
}
if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) {
perror("prctl");
exit(EXIT_FAILURE);
}
if (install_filter(strtol(argv[1], NULL, 0),
strtol(argv[2], NULL, 0)))
exit(EXIT_FAILURE);
execv(argv[3], &argv[3]);
perror("execv");
exit(EXIT_FAILURE);
}
^ permalink raw reply
* Re: [PATCH V2] powerpc/syscall: Fix seccomp errno handling with GENERIC_ENTRY
From: Mukesh Kumar Chaurasiya @ 2026-07-01 6:27 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: maddy, mpe, npiggin, chleroy, mkchauras, ryan.roberts, ruanjinjie,
linuxppc-dev, linux-kernel, Michal Suchánek
In-Reply-To: <a58675e4-f5a6-4a21-9871-904aad33b683@linux.ibm.com>
On Wed, Jul 01, 2026 at 01:41:09AM +0530, Shrikanth Hegde wrote:
> Hi Mukesh.
>
> On 6/29/26 11:59 PM, Mukesh Kumar Chaurasiya (IBM) wrote:
> > After enabling GENERIC_ENTRY on PowerPC, seccomp filters using
> > SCMP_ACT_ERRNO without an explicit errnoRet value return ENOSYS
> > (Function not implemented) instead of the expected EPERM (Operation
> > not permitted).
> >
> > The issue occurs in system_call_exception() when syscall_enter_from_user_mode()
> > returns -1 to indicate the syscall should be skipped (e.g., blocked by seccomp).
> > The current code treats this -1 as a syscall number and compares it against
> > NR_syscalls. Since -1 is greater than NR_syscalls,
> > the code incorrectly returns -ENOSYS, overwriting the errno that seccomp
> > already set via syscall_set_return_value().
> >
> > The generic entry code in syscall_trace_enter() calls __secure_computing(),
> > which sets the appropriate errno in regs->gpr[3] and returns -1 to signal
> > that the syscall should be skipped. However, the PowerPC syscall handler
> > was not checking for this -1 return value before validating the syscall
> > number.
> >
> > Fix this by explicitly checking if syscall_enter_from_user_mode() returns
> > -1 and returning the value already set in regs->gpr[3] (the errno from
> > seccomp) before performing the syscall number validation.
> >
> > Also Move the syscall_enter_from_user_mode() call and the seccomp/ptrace
> > skip check to after the NR_syscalls bounds check.
> >
> > When syscall -1 was passed, the r0 == -1L check would trigger before
> > the NR_syscalls check, causing syscall_get_error() to return 0 instead
> > of -ENOSYS. This resulted in a silent success (ret=0, errno=0) instead
> > of the expected ENOSYS error.
> >
> > By moving syscall_enter_from_user_mode() after the bounds check, an
> > initial syscall number of -1 is correctly rejected with -ENOSYS first.
> > The seccomp/ptrace skip path still works correctly for valid syscall
> > numbers that get overridden to -1 by seccomp or ptrace.
> >
> > This aligns PowerPC's behavior with other architectures using GENERIC_ENTRY
> > and restores correct seccomp errno handling.
> >
> > Fixes: bee25f97ad24 ("powerpc: Enable GENERIC_ENTRY feature")
> > Reported-by: Michal Suchánek <msuchanek@suse.de>
> > Closes: https://lore.kernel.org/all/ajpp-_XnbF3UTM_E@kunlun.suse.cz/
> > Signed-off-by: Mukesh Kumar Chaurasiya (IBM) <mkchauras@gmail.com>
> > ---
> >
> > v1 -> v2:
> > - Fix issues in the previous fix (Michal)
> > v1: https://lore.kernel.org/all/20260624171520.772408-1-mkchauras@gmail.com
> >
> > arch/powerpc/kernel/syscall.c | 7 ++++++-
> > 1 file changed, 6 insertions(+), 1 deletion(-)
> >
> > diff --git a/arch/powerpc/kernel/syscall.c b/arch/powerpc/kernel/syscall.c
> > index a9da2af6efa8..36d73933a311 100644
> > --- a/arch/powerpc/kernel/syscall.c
> > +++ b/arch/powerpc/kernel/syscall.c
> > @@ -20,7 +20,6 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
> > syscall_fn f;
> > add_random_kstack_offset();
> > - r0 = syscall_enter_from_user_mode(regs, r0);
> > if (unlikely(r0 >= NR_syscalls)) {
> > if (unlikely(trap_is_unsupported_scv(regs))) {
> > @@ -31,6 +30,12 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
> > return -ENOSYS;
> > }
> > + r0 = syscall_enter_from_user_mode(regs, r0);
> > +
>
> I see many arch first do syscall_enter_from_user_mode and then check for return value.
> take x86 for example,
>
> __visible noinstr bool do_syscall_64(struct pt_regs *regs, int nr)
> {
> nr = syscall_enter_from_user_mode(regs, nr);
>
> if (!do_syscall_x64(regs, nr) && !do_syscall_x32(regs, nr) && nr != -1) {
> /* Invalid system call, but still a system call. */
> regs->ax = __x64_sys_ni_syscall(regs);
> }
>
> }
>
> So seccomp fails silently there if initial nr was -1?
>
Hey,
No the -1 syscall ignores the error silently and returns 0.
From the above snippet from x86. Out behaviour also will remain same.
The reasoning for that i have given here
https://lore.kernel.org/all/akKnSqGUeFIGOfHb@li-1a3e774c-28e4-11b2-a85c-acc9f2883e29.ibm.com/
Regards,
Mukesh
>
>
> > + /* Seccomp or ptrace may have set return value, skip syscall */
> > + if (unlikely(r0 == -1L))
> > + return syscall_get_error(current, regs);
> > +
> > /* May be faster to do array_index_nospec? */
> > barrier_nospec();
>
> Code per se, looks okay to me.
^ permalink raw reply
* Re: [PATCH v3 0/8] treewide: remove unnecessary invalid range checks in memblock iteration loops
From: Mike Rapoport @ 2026-07-01 6:22 UTC (permalink / raw)
To: Albert Ou, Andrew Morton, Andrey Ryabinin, Catalin Marinas,
Huacai Chen, Madhavan Srinivasan, Michael Ellerman, Muchun Song,
Oscar Salvador, Palmer Dabbelt, Paul Walmsley, Russell King,
Will Deacon, Sang-Heon Jeon
Cc: linux-mm, Alexander Potapenko, Alexandre Ghiti, Andrey Konovalov,
Christophe Leroy (CS GROUP), David Hildenbrand, Dmitry Vyukov,
kasan-dev, linux-arm-kernel, linux-kernel, linuxppc-dev,
linux-riscv, loongarch, Nicholas Piggin, Vincenzo Frascino,
WANG Xuerui
In-Reply-To: <20260630150413.1718632-1-ekffu200098@gmail.com>
On Wed, 01 Jul 2026 00:04:05 +0900, Sang-Heon Jeon wrote:
> treewide: remove unnecessary invalid range checks in memblock iteration loops
>
> The memblock API guarantees that for_each_mem_range() and
> for_each_mem_pfn_range() never return an invalid range, meaning start is
> always less than end.
>
> Several memblock callers still have unnecessary invalid range checks in
> their loop bodies, so remove them.
>
> [...]
Applied to range-checks branch of memblock.git tree, thanks!
[1/8] arm64: mm: remove unreachable invalid range check in kasan_init_shadow()
commit: f6e6c57f2e100813be6d5882060fef81cb6f32de
[2/8] LoongArch: remove unreachable invalid range check in kasan_init()
commit: 68d3b7d58237b8794d9343210d4c90b8381cfa96
[3/8] riscv: remove unreachable invalid range check in create_linear_mapping_page_table()
commit: 99e60ebe108817fd48791f2cb81f1e672ad8285b
[4/8] riscv: remove unreachable invalid range check in kasan_init()
commit: 8008f6995d0c0cf22e7b4d60e95fe14085c10f35
[5/8] ARM: remove unreachable invalid range check in kasan_init()
commit: 3db1a9ccf9290ea3fe01aadf3bd743ef7032b24a
[6/8] powerpc64/kasan: Remove unreachable invalid range check in kasan_init_phys_region()
commit: 21e95a4b6f34770571eeeaa6adb535efba1b7281
[7/8] mm: remove unnecessary empty range check in early_calculate_totalpages()
commit: c6af91e48594191f62ad3c1fcc8269b1eb539ef8
[8/8] mm/hugetlb: remove unnecessary empty range check in hugetlb_bootmem_set_nodes()
commit: 2db44dc8bdd3cfc933a3193724f4d38b03e35b80
tree: https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock
branch: range-checks
--
Sincerely yours,
Mike.
^ permalink raw reply
* Re: [PATCH 1/1] KVM: powerpc/book3s_hv: Use generic xfer to guest work function
From: Shrikanth Hegde @ 2026-07-01 6:13 UTC (permalink / raw)
To: Vishal Chourasia
Cc: npiggin, mpe, chleroy, gautam, bigeasy, linuxppc-dev, kvm,
linux-kernel, maddy
In-Reply-To: <20260626105449.2897924-4-vishalc@linux.ibm.com>
Hi Vishal,
On 6/26/26 4:23 PM, Vishal Chourasia wrote:
> Use the generic infrastructure to check for and handle pending work
> before transitioning into guest mode, replacing the open-coded
> need_resched() and cond_resched() checks.
>
> This picks up handling for TIF_NOTIFY_RESUME, which was previously
> ignored, meaning task work will now be correctly handled on every
> guest re-entry.
It would indeed be good if powerpc moves go generic VIRT_XFER_TO_GUEST_WORK.
It does take care of RESUME.
In addition today, I doubt powerpc kvm works well for LAZY preemption.
generic infra will take care of it too.
>
> Signed-off-by: Vishal Chourasia <vishalc@linux.ibm.com>
> ---
> arch/powerpc/kvm/Kconfig | 1 +
> arch/powerpc/kvm/book3s_hv.c | 58 +++++++++++++++++++++++++++++++-----
> 2 files changed, 52 insertions(+), 7 deletions(-)
>
> diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig
> index 9a0d1c1aca6c..36aec58c5f22 100644
> --- a/arch/powerpc/kvm/Kconfig
> +++ b/arch/powerpc/kvm/Kconfig
> @@ -81,6 +81,7 @@ config KVM_BOOK3S_64_HV
> depends on KVM_BOOK3S_64 && PPC_POWERNV
> select KVM_BOOK3S_HV_POSSIBLE
> select KVM_BOOK3S_HV_PMU
> + select VIRT_XFER_TO_GUEST_WORK
This takes care of HV only.
Does PR/booke run into the same problem?
Does anyone out there who runs them still and care about cgroup bandwidth control mechanism on it?
It may be worth adding comment that PR still runs into the same issue.
> select CMA
> help
> Support running unmodified book3s_64 guest kernels in
> diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
> index 61dbeea317f3..b012512342e6 100644
> --- a/arch/powerpc/kvm/book3s_hv.c
> +++ b/arch/powerpc/kvm/book3s_hv.c
> @@ -3850,10 +3850,20 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc)
> * and return without going into the guest(s).
> * If the mmu_ready flag has been cleared, don't go into the
> * guest because that means a HPT resize operation is in progress.
> + *
> + * xfer_to_guest_mode_work_pending() is the IRQs-disabled recheck for
> + * pending guest-mode work (reschedule, signals, and TIF_NOTIFY_RESUME
> + * task_work such as the deferred CFS throttle). It is the pre-POWER9
> + * analog of the final gate in kvmhv_run_single_vcpu(), and a superset
> + * of the old need_resched() check: it catches work that raced in after
> + * the drain in kvmppc_run_vcpu(), so a CPU-bound vCPU is throttled here
> + * instead of running one more guest dispatch past its quota. IRQs are
> + * hard-disabled just above, so the non-__ variant (which asserts that)
> + * is the correct one.
> */
> local_irq_disable();
> hard_irq_disable();
> - if (lazy_irq_pending() || need_resched() ||
> + if (lazy_irq_pending() || xfer_to_guest_mode_work_pending() ||
> recheck_signals_and_mmu(&core_info)) {
> local_irq_enable();
> vc->vcore_state = VCORE_INACTIVE;
> @@ -4824,10 +4834,24 @@ static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu)
> vc->runner = vcpu;
> if (n_ceded == vc->n_runnable) {
> kvmppc_vcore_blocked(vc);
> - } else if (need_resched()) {
> + } else if (__xfer_to_guest_mode_work_pending()) {
> kvmppc_vcore_preempt(vc);
> - /* Let something else run */
> - cond_resched_lock(&vc->lock);
> + /*
> + * Let something else run, and run pending guest-mode
> + * work (reschedule, and TIF_NOTIFY_RESUME task_work such
> + * as the deferred CFS throttle) before we would re-enter
> + * the guest, so a CPU-bound vCPU is actually throttled
> + * here instead of running past its quota. This is a
> + * superset of the old need_resched() check. Use the raw
> + * helper, not the kvm_ wrapper: signals (KVM_EXIT_INTR
> + * and the signal_exits stat) are accounted by this path's
> + * existing handling below, so going through the wrapper
> + * here would double-count them. The helper may schedule(),
> + * so the vcore lock is dropped around it.
> + */
> + spin_unlock(&vc->lock);
> + xfer_to_guest_mode_handle_work();
> + spin_lock(&vc->lock);
> if (vc->vcore_state == VCORE_PREEMPT)
> kvmppc_vcore_end_preempt(vc);
> } else {
> @@ -4899,8 +4923,21 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,
> }
> }
>
> - if (need_resched())
> - cond_resched();
> + /*
> + * Run pending work before (re-)entering the guest, most importantly
> + * task_work queued via TWA_RESUME (e.g. the deferred CFS bandwidth
> + * throttle, which only sets TIF_NOTIFY_RESUME). Without this a CPU-bound
> + * vCPU that keeps returning RESUME_GUEST never reaches an exit-to-user
> + * point, so the throttle is never enforced and the task runs far beyond
> + * its quota. The helper also handles reschedule and signals, replacing
> + * the cond_resched() that was here. It may schedule(), so it runs before
> + * preemption and IRQs are disabled, with no vcore/KVM locks held. This
> + * is the per-reentry site shared by the bare-metal and pseries (nested)
> + * paths, so both are covered.
> + */
> + r = kvm_xfer_to_guest_mode_handle_work(vcpu);
> + if (r) /* -EINTR: signal pending, exit to userspace (KVM_EXIT_INTR) */
> + return r;
>
> kvmppc_update_vpas(vcpu);
>
> @@ -4916,7 +4953,14 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit,
>
> if (signal_pending(current))
> goto sigpend;
xfer_to_guest_mode_work_pending checks for signals too right?
#define XFER_TO_GUEST_MODE_WORK \
(_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | _TIF_SIGPENDING | \
_TIF_NOTIFY_SIGNAL | _TIF_NOTIFY_RESUME | \
ARCH_XFER_TO_GUEST_MODE_WORK)
> - if (need_resched() || !kvm->arch.mmu_ready)
> + /*
> + * Re-check for pending guest-mode work with IRQs disabled, to catch
> + * anything (e.g. a TIF_NOTIFY_RESUME task_work such as the deferred CFS
> + * throttle) that raced in after the check above. Bail back to the outer
> + * loop, which re-enters here and runs the work. This is a superset of
> + * the previous need_resched() check.
> + */
> + if (xfer_to_guest_mode_work_pending() || !kvm->arch.mmu_ready)
> goto out;
>
> vcpu->cpu = pcpu;
Copying from the discussion thread at that time,
"""
on x86:
kvm_arch_vcpu_ioctl_run
vcpu_run
for () {
.. run guest..
xfer_to_guest_mode_handle_work
schedule
}
on Powerpc: ( taking book3s_hv flavour):
kvm_arch_vcpu_ioctl_run
kvmppc_vcpu_run_hv *1
do while() {
kvmhv_run_single_vcpu or kvmppc_run_vcpu
-- checking for need_resched and signals and bails out *2
}
*1 - checks for need resched and signals before entering guest
*2 - checks for need resched and signals while running the guest
This patch is addressing only *1 but it needs to address *2 as well using generic framework.
I think it is doable for books3s_hv atleast. (though might need rewrite)
"""
A few questions that comes to my mind.
1. I think you are doing for both *1 and *2 right? Is *1 necessary still for powerpc?
2. There is a for loop. What is it doing? Does this still need similar checks?
if (is_kvmppc_resume_guest(r) && !kvmppc_vcpu_check_block(vcpu)) {
kvmppc_set_timer(vcpu);
prepare_to_rcuwait(wait);
for (;;) {
set_current_state(TASK_INTERRUPTIBLE);
if (signal_pending(current)) {
vcpu->stat.signal_exits++;
run->exit_reason = KVM_EXIT_INTR;
vcpu->arch.ret = -EINTR;
break;
}
if (kvmppc_vcpu_check_block(vcpu))
break;
trace_kvmppc_vcore_blocked(vcpu, 0);
schedule();
trace_kvmppc_vcore_blocked(vcpu, 1);
}
finish_rcuwait(wait);
}
vcpu->arch.ceded = 0;
PS: cc'ing me would have helped me to see the mail earlier since you had referred
to the patch I had sent.
^ permalink raw reply
* [PATCH v7 22/22] swiotlb: remove unused SWIOTLB_FORCE flag
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
SWIOTLB_FORCE has no remaining in-tree users. Forced bouncing is now
controlled through the swiotlb=force command line option via
swiotlb_force_bounce.
Remove the unused flag and simplify the force_bounce initialization.
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
include/linux/swiotlb.h | 3 +--
kernel/dma/swiotlb.c | 3 +--
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index c3bf7ed6f7a6..9caca923c380 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -15,8 +15,7 @@ struct page;
struct scatterlist;
#define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */
-#define SWIOTLB_FORCE (1 << 1) /* force bounce buffering */
-#define SWIOTLB_ANY (1 << 2) /* allow any memory for the buffer */
+#define SWIOTLB_ANY (1 << 1) /* allow any memory for the buffer */
/*
* Maximum allowable number of contiguous slabs to map,
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 8b7e47504304..897aba538c5b 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -400,8 +400,7 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
if (swiotlb_force_disable)
return;
- io_tlb_default_mem.force_bounce =
- swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
+ io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
#ifdef CONFIG_SWIOTLB_DYNAMIC
if (!remap)
--
2.43.0
^ permalink raw reply related
* [PATCH v7 21/22] dma: swiotlb: handle set_memory_decrypted() failures
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
Check the return value when converting swiotlb pools between encrypted and
decrypted mappings. If the default pool cannot be decrypted after early
initialization, mark the pool fully used so it cannot satisfy future bounce
allocations.
For late initialization, return the `set_memory_decrypted()` failure. For
restricted DMA pools, fail device initialization if the reserved pool
cannot be decrypted.
This prevents swiotlb from using pools whose encryption attributes do not
match their metadata, and avoids returning pages with uncertain encryption
state back to the allocator.
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/swiotlb.c | 80 +++++++++++++++++++++++++++++++++++---------
1 file changed, 65 insertions(+), 15 deletions(-)
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 4d0f2c04d891..8b7e47504304 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -248,6 +248,23 @@ static inline unsigned long nr_slots(u64 val)
return DIV_ROUND_UP(val, IO_TLB_SIZE);
}
+static void swiotlb_mark_pool_used(struct io_tlb_pool *pool)
+{
+ unsigned long i;
+
+ for (i = 0; i < pool->nareas; i++) {
+ pool->areas[i].index = 0;
+ pool->areas[i].used = pool->area_nslabs;
+ }
+
+ for (i = 0; i < pool->nslabs; i++) {
+ pool->slots[i].list = 0;
+ pool->slots[i].orig_addr = INVALID_PHYS_ADDR;
+ pool->slots[i].alloc_size = 0;
+ pool->slots[i].pad_slots = 0;
+ }
+}
+
/*
* Early SWIOTLB allocation may be too early to allow an architecture to
* perform the desired operations. This function allows the architecture to
@@ -272,8 +289,16 @@ void __init swiotlb_update_mem_attributes(void)
return;
bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
- if (io_tlb_default_mem.cc_shared)
- set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
+ if (io_tlb_default_mem.cc_shared) {
+ int ret;
+
+ ret = set_memory_decrypted((unsigned long)mem->vaddr,
+ bytes >> PAGE_SHIFT);
+ if (ret) {
+ pr_warn("Failed to decrypt default memory pool, disabling it\n");
+ swiotlb_mark_pool_used(mem);
+ }
+ }
}
static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
@@ -442,9 +467,10 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
{
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
+ unsigned int order, area_order, slot_order;
+ bool leak_pages = false;
unsigned int nareas;
unsigned char *vstart = NULL;
- unsigned int order, area_order;
bool retried = false;
int rc = 0;
@@ -504,6 +530,7 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
(PAGE_SIZE << order) >> 20);
}
+ rc = -ENOMEM;
nareas = limit_nareas(default_nareas, nslabs);
area_order = get_order(array_size(sizeof(*mem->areas), nareas));
mem->areas = (struct io_tlb_area *)
@@ -511,14 +538,20 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
if (!mem->areas)
goto error_area;
+ slot_order = get_order(array_size(sizeof(*mem->slots), nslabs));
mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
- get_order(array_size(sizeof(*mem->slots), nslabs)));
+ slot_order);
if (!mem->slots)
goto error_slots;
- if (io_tlb_default_mem.cc_shared)
- set_memory_decrypted((unsigned long)vstart,
- (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+ if (io_tlb_default_mem.cc_shared) {
+ rc = set_memory_decrypted((unsigned long)vstart,
+ (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+ if (rc) {
+ leak_pages = true;
+ goto error_decrypt;
+ }
+ }
swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), vstart, nslabs, true,
nareas);
@@ -527,16 +560,20 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
swiotlb_print_info();
return 0;
+error_decrypt:
+ free_pages((unsigned long)mem->slots, slot_order);
error_slots:
free_pages((unsigned long)mem->areas, area_order);
error_area:
- free_pages((unsigned long)vstart, order);
- return -ENOMEM;
+ if (!leak_pages)
+ free_pages((unsigned long)vstart, order);
+ return rc;
}
void __init swiotlb_exit(void)
{
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
+ bool leak_pages = false;
unsigned long tbl_vaddr;
size_t tbl_size, slots_size;
unsigned int area_order;
@@ -552,19 +589,23 @@ void __init swiotlb_exit(void)
tbl_size = PAGE_ALIGN(mem->end - mem->start);
slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
- if (io_tlb_default_mem.cc_shared)
- set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+ if (io_tlb_default_mem.cc_shared) {
+ if (set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT))
+ leak_pages = true;
+ }
if (mem->late_alloc) {
area_order = get_order(array_size(sizeof(*mem->areas),
mem->nareas));
free_pages((unsigned long)mem->areas, area_order);
- free_pages(tbl_vaddr, get_order(tbl_size));
+ if (!leak_pages)
+ free_pages(tbl_vaddr, get_order(tbl_size));
free_pages((unsigned long)mem->slots, get_order(slots_size));
} else {
memblock_free(mem->areas,
array_size(sizeof(*mem->areas), mem->nareas));
- memblock_phys_free(mem->start, tbl_size);
+ if (!leak_pages)
+ memblock_phys_free(mem->start, tbl_size);
memblock_free(mem->slots, slots_size);
}
@@ -1957,9 +1998,18 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
* restricted mem pool is shared by default
*/
if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
+ int ret;
+
mem->cc_shared = true;
- set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
- rmem->size >> PAGE_SHIFT);
+ ret = set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
+ rmem->size >> PAGE_SHIFT);
+ if (ret) {
+ dev_err(dev, "Failed to decrypt restricted DMA pool\n");
+ kfree(pool->areas);
+ kfree(pool->slots);
+ kfree(mem);
+ return ret;
+ }
} else {
mem->cc_shared = false;
}
--
2.43.0
^ permalink raw reply related
* [PATCH v7 20/22] dma: swiotlb: free dynamic pools from process context
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
swiotlb_dyn_free() is used after removing a dynamic swiotlb pool from
RCU-protected lists. It can call swiotlb_free_tlb(), which may need to
restore the encryption state of an unencrypted pool with
set_memory_encrypted() before freeing the pages.
RCU callbacks run in atomic context, but set_memory_encrypted() is not
guaranteed to be atomic-safe on all architectures. For example, page
attribute updates may allocate page tables or take sleeping locks.
Use queue_rcu_work() for dynamic pool freeing instead. This keeps the RCU
grace period before freeing a published pool, while running the actual pool
teardown from workqueue context. Use the same helper for the transient-pool
error path, since that path may also be reached from atomic DMA mapping
context.
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
include/linux/swiotlb.h | 4 ++--
kernel/dma/swiotlb.c | 19 +++++++++++--------
2 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index ee42f7588847..c3bf7ed6f7a6 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -64,7 +64,7 @@ extern void __init swiotlb_update_mem_attributes(void);
* @areas: Array of memory area descriptors.
* @slots: Array of slot descriptors.
* @node: Member of the IO TLB memory pool list.
- * @rcu: RCU head for swiotlb_dyn_free().
+ * @dyn_free: RCU work item used to free the pool from process context.
* @transient: %true if transient memory pool.
* @cc_shared: %true if the pool memory is shared for confidential computing.
*/
@@ -80,7 +80,7 @@ struct io_tlb_pool {
struct io_tlb_slot *slots;
#ifdef CONFIG_SWIOTLB_DYNAMIC
struct list_head node;
- struct rcu_head rcu;
+ struct rcu_work dyn_free;
bool transient;
bool cc_shared;
#endif
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index b5960c2e98d8..4d0f2c04d891 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -779,13 +779,10 @@ static void swiotlb_dyn_alloc(struct work_struct *work)
add_mem_pool(mem, pool);
}
-/**
- * swiotlb_dyn_free() - RCU callback to free a memory pool
- * @rcu: RCU head in the corresponding struct io_tlb_pool.
- */
-static void swiotlb_dyn_free(struct rcu_head *rcu)
+static void swiotlb_dyn_free_work(struct work_struct *work)
{
- struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);
+ struct io_tlb_pool *pool =
+ container_of(to_rcu_work(work), struct io_tlb_pool, dyn_free);
size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
size_t tlb_size = pool->end - pool->start;
@@ -794,6 +791,12 @@ static void swiotlb_dyn_free(struct rcu_head *rcu)
kfree(pool);
}
+static void swiotlb_schedule_dyn_free(struct io_tlb_pool *pool)
+{
+ INIT_RCU_WORK(&pool->dyn_free, swiotlb_dyn_free_work);
+ queue_rcu_work(system_wq, &pool->dyn_free);
+}
+
/**
* __swiotlb_find_pool() - find the IO TLB pool for a physical address
* @dev: Device which has mapped the DMA buffer.
@@ -840,7 +843,7 @@ static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
list_del_rcu(&pool->node);
spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags);
- call_rcu(&pool->rcu, swiotlb_dyn_free);
+ swiotlb_schedule_dyn_free(pool);
}
#endif /* CONFIG_SWIOTLB_DYNAMIC */
@@ -1281,7 +1284,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
index = swiotlb_search_pool_area(dev, pool, 0, orig_addr, tbl_dma_addr,
alloc_size, alloc_align_mask);
if (index < 0) {
- swiotlb_dyn_free(&pool->rcu);
+ swiotlb_schedule_dyn_free(pool);
return -1;
}
--
2.43.0
^ permalink raw reply related
* [PATCH v7 19/22] dma-direct: rename ret to cpu_addr in alloc helpers
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
ret in dma_direct_alloc() and dma_direct_alloc_pages() holds the returned
CPU mapping, not a generic return value. Rename it to cpu_addr and update
the remaining uses to match.
This makes the allocation paths easier to follow and keeps the local naming
consistent with what the variable actually represents.
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 40 ++++++++++++++++++++--------------------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 23138519cf22..9575d68571bf 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -204,7 +204,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
bool mark_mem_decrypt = false;
bool allow_highmem = true;
struct page *page;
- void *ret;
+ void *cpu_addr;
if (force_dma_unencrypted(dev))
attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
@@ -261,9 +261,10 @@ void *dma_direct_alloc(struct device *dev, size_t size,
*/
if ((remap || (attrs & __DMA_ATTR_ALLOC_CC_SHARED)) &&
dma_direct_use_pool(dev, gfp)) {
- page = dma_direct_alloc_from_pool(dev, size, dma_handle,
- &ret, gfp, attrs);
- return page ? ret : NULL;
+ page = dma_direct_alloc_from_pool(dev, size,
+ dma_handle, &cpu_addr,
+ gfp, attrs);
+ return page ? cpu_addr : NULL;
}
if (is_swiotlb_for_alloc(dev)) {
@@ -310,34 +311,33 @@ void *dma_direct_alloc(struct device *dev, size_t size,
arch_dma_prep_coherent(page, size);
/* create a coherent mapping */
- ret = dma_common_contiguous_remap(page, size, prot,
- __builtin_return_address(0));
- if (!ret)
+ cpu_addr = dma_common_contiguous_remap(page, size, prot,
+ __builtin_return_address(0));
+ if (!cpu_addr)
goto out_encrypt_pages;
} else {
- ret = page_address(page);
+ cpu_addr = page_address(page);
}
- memset(ret, 0, size);
+ memset(cpu_addr, 0, size);
if (set_uncached) {
void *uncached_cpu_addr;
arch_dma_prep_coherent(page, size);
- uncached_cpu_addr = arch_dma_set_uncached(ret, size);
+ uncached_cpu_addr = arch_dma_set_uncached(cpu_addr, size);
if (IS_ERR(uncached_cpu_addr))
goto out_free_remap_pages;
- ret = uncached_cpu_addr;
+ cpu_addr = uncached_cpu_addr;
}
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
!!(attrs & __DMA_ATTR_ALLOC_CC_SHARED));
- return ret;
-
+ return cpu_addr;
out_free_remap_pages:
if (remap)
- dma_common_free_remap(ret, size);
+ dma_common_free_remap(cpu_addr, size);
out_encrypt_pages:
if (mark_mem_decrypt &&
@@ -429,21 +429,21 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
{
unsigned long attrs = 0;
struct page *page;
- void *ret;
+ void *cpu_addr;
if (force_dma_unencrypted(dev))
attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle,
- &ret, gfp, attrs);
+ &cpu_addr, gfp, attrs);
if (is_swiotlb_for_alloc(dev)) {
page = dma_direct_alloc_swiotlb(dev, size, attrs);
if (!page)
return NULL;
- ret = page_address(page);
+ cpu_addr = page_address(page);
goto setup_page;
}
@@ -451,12 +451,12 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
if (!page)
return NULL;
- ret = page_address(page);
+ cpu_addr = page_address(page);
if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) &&
- dma_set_decrypted(dev, ret, size))
+ dma_set_decrypted(dev, cpu_addr, size))
goto out_leak_pages;
setup_page:
- memset(ret, 0, size);
+ memset(cpu_addr, 0, size);
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
!!(attrs & __DMA_ATTR_ALLOC_CC_SHARED));
return page;
--
2.43.0
^ permalink raw reply related
* [PATCH v7 18/22] dma-direct: select DMA address encoding from __DMA_ATTR_ALLOC_CC_SHARED
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko,
Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
Make the dma-direct helpers derive the DMA address encoding from
__DMA_ATTR_ALLOC_CC_SHARED instead of implicitly relying on
force_dma_unencrypted() inside phys_to_dma_direct()
Pass an explicit unencrypted/decrypted state into phys_to_dma_direct(),
make the alloc paths return DMA addresses that match the requested buffer
encryption state. Also only call dma_set_decrypted() when
__DMA_ATTR_ALLOC_CC_SHARED is actually set.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 43 ++++++++++++++++++++++++++-----------------
1 file changed, 26 insertions(+), 17 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 964e8b0d8709..23138519cf22 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -24,11 +24,11 @@
u64 zone_dma_limit __ro_after_init = DMA_BIT_MASK(24);
static inline dma_addr_t phys_to_dma_direct(struct device *dev,
- phys_addr_t phys)
+ phys_addr_t phys, bool unencrypted)
{
- if (force_dma_unencrypted(dev))
+ if (unencrypted)
return phys_to_dma_unencrypted(dev, phys);
- return phys_to_dma(dev, phys);
+ return phys_to_dma_encrypted(dev, phys);
}
static inline struct page *dma_direct_to_page(struct device *dev,
@@ -39,8 +39,9 @@ static inline struct page *dma_direct_to_page(struct device *dev,
u64 dma_direct_get_required_mask(struct device *dev)
{
+ bool require_decrypted = force_dma_unencrypted(dev);
phys_addr_t phys = ((phys_addr_t)max_pfn << PAGE_SHIFT) - 1;
- u64 max_dma = phys_to_dma_direct(dev, phys);
+ u64 max_dma = phys_to_dma_direct(dev, phys, require_decrypted);
return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
}
@@ -69,7 +70,8 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit)
bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
{
- dma_addr_t dma_addr = phys_to_dma_direct(dev, phys);
+ bool require_decrypted = force_dma_unencrypted(dev);
+ dma_addr_t dma_addr = phys_to_dma_direct(dev, phys, require_decrypted);
if (dma_addr == DMA_MAPPING_ERROR)
return false;
@@ -79,17 +81,18 @@ bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)
static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size)
{
- if (!force_dma_unencrypted(dev))
- return 0;
- return set_memory_decrypted((unsigned long)vaddr, PFN_UP(size));
+ int ret;
+
+ ret = set_memory_decrypted((unsigned long)vaddr, PFN_UP(size));
+ if (ret)
+ pr_warn_ratelimited("leaking DMA memory that can't be decrypted\n");
+ return ret;
}
static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)
{
int ret;
- if (!force_dma_unencrypted(dev))
- return 0;
ret = set_memory_encrypted((unsigned long)vaddr, PFN_UP(size));
if (ret)
pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n");
@@ -169,7 +172,8 @@ static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size,
dma_coherent_ok);
if (!page)
return NULL;
- *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
+ *dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
+ !!(attrs & __DMA_ATTR_ALLOC_CC_SHARED));
return page;
}
@@ -185,9 +189,11 @@ static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,
/* remove any dirty cache lines on the kernel alias */
if (!PageHighMem(page))
arch_dma_prep_coherent(page, size);
-
- /* return the page pointer as the opaque cookie */
- *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
+ /*
+ * return the page pointer as the opaque cookie.
+ * Never used for unencrypted allocation
+ */
+ *dma_handle = phys_to_dma_encrypted(dev, page_to_phys(page));
return page;
}
@@ -324,7 +330,8 @@ void *dma_direct_alloc(struct device *dev, size_t size,
ret = uncached_cpu_addr;
}
- *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
+ *dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
+ !!(attrs & __DMA_ATTR_ALLOC_CC_SHARED));
return ret;
@@ -445,11 +452,13 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
return NULL;
ret = page_address(page);
- if (dma_set_decrypted(dev, ret, size))
+ if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) &&
+ dma_set_decrypted(dev, ret, size))
goto out_leak_pages;
setup_page:
memset(ret, 0, size);
- *dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
+ *dma_handle = phys_to_dma_direct(dev, page_to_phys(page),
+ !!(attrs & __DMA_ATTR_ALLOC_CC_SHARED));
return page;
out_leak_pages:
return NULL;
--
2.43.0
^ permalink raw reply related
* [PATCH v7 17/22] dma-direct: set decrypted flag for remapped DMA allocations
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko,
Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
Devices that are DMA non-coherent and require a remap were skipping
dma_set_decrypted(), leaving DMA buffers encrypted even when the device
requires unencrypted access. Move the call after the if (remap) branch
so that both the direct and remapped allocation paths correctly mark the
allocation as decrypted (or fail cleanly) before use.
Fix dma_direct_alloc() and dma_direct_free() to apply set_memory_*() to the
linear-map alias of the backing pages instead of the remapped CPU address.
Also disallow highmem pages for __DMA_ATTR_ALLOC_CC_SHARED, because highmem
buffers do not provide a usable linear-map address.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 56 +++++++++++++++++++++++++++++++++++----------
1 file changed, 44 insertions(+), 12 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index edf40746a775..964e8b0d8709 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -196,14 +196,23 @@ void *dma_direct_alloc(struct device *dev, size_t size,
{
bool remap = false, set_uncached = false;
bool mark_mem_decrypt = false;
+ bool allow_highmem = true;
struct page *page;
void *ret;
if (force_dma_unencrypted(dev))
attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
- if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
+ if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) {
+ /*
+ * Unencrypted/shared DMA requires a linear-mapped buffer
+ * address to look up the PFN and set architecture-required PFN
+ * attributes. This is not possible with HighMem. Avoid HighMem
+ * allocation.
+ */
+ allow_highmem = false;
mark_mem_decrypt = true;
+ }
size = PAGE_ALIGN(size);
if (attrs & DMA_ATTR_NO_WARN)
@@ -265,7 +274,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
}
/* we always manually zero the memory once we are done */
- page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);
+ page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, allow_highmem);
if (!page)
return NULL;
@@ -280,6 +289,14 @@ void *dma_direct_alloc(struct device *dev, size_t size,
set_uncached = false;
}
+ if (mark_mem_decrypt) {
+ void *lm_addr;
+
+ lm_addr = page_address(page);
+ if (set_memory_decrypted((unsigned long)lm_addr, PFN_UP(size)))
+ goto out_leak_pages;
+ }
+
if (remap) {
pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
@@ -290,29 +307,36 @@ void *dma_direct_alloc(struct device *dev, size_t size,
ret = dma_common_contiguous_remap(page, size, prot,
__builtin_return_address(0));
if (!ret)
- goto out_free_pages;
+ goto out_encrypt_pages;
} else {
ret = page_address(page);
- if (mark_mem_decrypt && dma_set_decrypted(dev, ret, size))
- goto out_leak_pages;
}
memset(ret, 0, size);
if (set_uncached) {
+ void *uncached_cpu_addr;
+
arch_dma_prep_coherent(page, size);
- ret = arch_dma_set_uncached(ret, size);
- if (IS_ERR(ret))
- goto out_encrypt_pages;
+ uncached_cpu_addr = arch_dma_set_uncached(ret, size);
+ if (IS_ERR(uncached_cpu_addr))
+ goto out_free_remap_pages;
+ ret = uncached_cpu_addr;
}
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
return ret;
+
+out_free_remap_pages:
+ if (remap)
+ dma_common_free_remap(ret, size);
+
out_encrypt_pages:
- if (mark_mem_decrypt && dma_set_encrypted(dev, page_address(page), size))
- return NULL;
-out_free_pages:
+ if (mark_mem_decrypt &&
+ dma_set_encrypted(dev, page_address(page), size))
+ goto out_leak_pages;
+
if (!swiotlb_free(dev, page, size))
dma_free_contiguous(dev, page, size);
return NULL;
@@ -375,8 +399,16 @@ void dma_direct_free(struct device *dev, size_t size,
} else {
if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED))
arch_dma_clear_uncached(cpu_addr, size);
- if (mark_mem_encrypted && dma_set_encrypted(dev, cpu_addr, size))
+ }
+
+ if (mark_mem_encrypted) {
+ void *lm_addr;
+
+ lm_addr = phys_to_virt(phys);
+ if (set_memory_encrypted((unsigned long)lm_addr, PFN_UP(size))) {
+ pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n");
return;
+ }
}
if (swiotlb_pool)
--
2.43.0
^ permalink raw reply related
* [PATCH v7 16/22] dma-direct: make dma_direct_map_phys() honor DMA_ATTR_CC_SHARED
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko,
Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
Teach dma_direct_map_phys() to select the DMA address encoding based on
DMA_ATTR_CC_SHARED.
Use phys_to_dma_unencrypted() for decrypted mappings and
phys_to_dma_encrypted() otherwise. If a device requires unencrypted DMA
but the source physical address is still encrypted, force the mapping
through swiotlb so the DMA address and backing memory attributes remain
consistent.
Update the arm64, x86, s390 and powerpc secure-guest setup to not use
swiotlb force option
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
Changes from v3:
* Handle DMA_ATTR_MMIO
---
arch/arm64/mm/init.c | 4 +--
arch/powerpc/platforms/pseries/svm.c | 2 +-
arch/s390/mm/init.c | 2 +-
arch/x86/kernel/pci-dma.c | 4 +--
kernel/dma/direct.c | 4 ++-
kernel/dma/direct.h | 45 +++++++++++++++-------------
6 files changed, 31 insertions(+), 30 deletions(-)
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 97987f850a33..acf67c7064db 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -338,10 +338,8 @@ void __init arch_mm_preinit(void)
unsigned int flags = SWIOTLB_VERBOSE;
bool swiotlb = max_pfn > PFN_DOWN(arm64_dma_phys_limit);
- if (is_realm_world()) {
+ if (is_realm_world())
swiotlb = true;
- flags |= SWIOTLB_FORCE;
- }
if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb) {
/*
diff --git a/arch/powerpc/platforms/pseries/svm.c b/arch/powerpc/platforms/pseries/svm.c
index 384c9dc1899a..7a403dbd35ee 100644
--- a/arch/powerpc/platforms/pseries/svm.c
+++ b/arch/powerpc/platforms/pseries/svm.c
@@ -29,7 +29,7 @@ static int __init init_svm(void)
* need to use the SWIOTLB buffer for DMA even if dma_capable() says
* otherwise.
*/
- ppc_swiotlb_flags |= SWIOTLB_ANY | SWIOTLB_FORCE;
+ ppc_swiotlb_flags |= SWIOTLB_ANY;
/* Share the SWIOTLB buffer with the host. */
swiotlb_update_mem_attributes();
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 6b1c5a4fa9ce..8d1de5a2e554 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -166,7 +166,7 @@ static void __init pv_init(void)
virtio_set_mem_acc_cb(virtio_require_restricted_mem_acc);
/* make sure bounce buffers are shared */
- swiotlb_init(true, SWIOTLB_FORCE | SWIOTLB_VERBOSE);
+ swiotlb_init(true, SWIOTLB_VERBOSE);
swiotlb_update_mem_attributes();
}
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index 6267363e0189..75cf8f6ae8cd 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -59,10 +59,8 @@ static void __init pci_swiotlb_detect(void)
* bounce buffers as the hypervisor can't access arbitrary VM memory
* that is not explicitly shared with it.
*/
- if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
+ if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
x86_swiotlb_enable = true;
- x86_swiotlb_flags |= SWIOTLB_FORCE;
- }
}
#else
static inline void __init pci_swiotlb_detect(void)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 9935f7caa459..edf40746a775 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -693,8 +693,10 @@ size_t dma_direct_max_mapping_size(struct device *dev)
{
/* If SWIOTLB is active, use its maximum mapping size */
if (is_swiotlb_active(dev) &&
- (dma_addressing_limited(dev) || is_swiotlb_force_bounce(dev)))
+ (dma_addressing_limited(dev) || is_swiotlb_force_bounce(dev) ||
+ force_dma_unencrypted(dev)))
return swiotlb_max_mapping_size(dev);
+
return SIZE_MAX;
}
diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h
index e05dc7649366..f3fc28f352ba 100644
--- a/kernel/dma/direct.h
+++ b/kernel/dma/direct.h
@@ -88,37 +88,40 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev,
{
dma_addr_t dma_addr;
+ /*
+ * For a device requiring unencrypted DMA, MMIO memory is treated
+ * as shared by default.
+ */
+ if (force_dma_unencrypted(dev) && (attrs & DMA_ATTR_MMIO))
+ attrs |= DMA_ATTR_CC_SHARED;
+
if (is_swiotlb_force_bounce(dev)) {
- if (!(attrs & DMA_ATTR_CC_SHARED)) {
- if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
- return DMA_MAPPING_ERROR;
+ if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT))
+ return DMA_MAPPING_ERROR;
- return swiotlb_map(dev, phys, size, dir, attrs);
- }
- } else if (attrs & DMA_ATTR_CC_SHARED) {
- return DMA_MAPPING_ERROR;
+ return swiotlb_map(dev, phys, size, dir, attrs);
}
- if (attrs & DMA_ATTR_MMIO) {
- dma_addr = phys;
- if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs)))
- goto err_overflow;
- } else if (attrs & DMA_ATTR_CC_SHARED) {
+ if (attrs & DMA_ATTR_CC_SHARED)
dma_addr = phys_to_dma_unencrypted(dev, phys);
+ else
+ dma_addr = phys_to_dma_encrypted(dev, phys);
+
+ if (attrs & DMA_ATTR_MMIO) {
if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs)))
goto err_overflow;
- } else {
- dma_addr = phys_to_dma(dev, phys);
- if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) ||
- dma_kmalloc_needs_bounce(dev, size, dir)) {
- if (is_swiotlb_active(dev) &&
- !(attrs & DMA_ATTR_REQUIRE_COHERENT))
- return swiotlb_map(dev, phys, size, dir, attrs);
+ goto dma_mapped;
+ }
- goto err_overflow;
- }
+ if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) ||
+ dma_kmalloc_needs_bounce(dev, size, dir)) {
+ if (is_swiotlb_active(dev) &&
+ !(attrs & DMA_ATTR_REQUIRE_COHERENT))
+ return swiotlb_map(dev, phys, size, dir, attrs);
+ goto err_overflow;
}
+dma_mapped:
if (!dev_is_dma_coherent(dev) &&
!(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) {
arch_sync_dma_for_device(phys, size, dir);
--
2.43.0
^ permalink raw reply related
* [PATCH v7 15/22] dma-direct: pass attrs to dma_capable() for DMA_ATTR_CC_SHARED checks
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko,
Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
Teach dma_capable() about DMA_ATTR_CC_SHARED so the capability
check can reject encrypted DMA addresses for devices that require
unencrypted/shared DMA.
Also propagate DMA_ATTR_CC_SHARED in swiotlb_map() when the selected
SWIOTLB pool is decrypted so the capability check sees the correct DMA
address attribute.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
arch/x86/kernel/amd_gart_64.c | 30 ++++++++++++++++--------------
drivers/xen/swiotlb-xen.c | 6 +++---
include/linux/dma-direct.h | 10 +++++++++-
kernel/dma/direct.h | 6 +++---
kernel/dma/swiotlb.c | 2 +-
5 files changed, 32 insertions(+), 22 deletions(-)
diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c
index e8000a56732e..b5f1f031d45b 100644
--- a/arch/x86/kernel/amd_gart_64.c
+++ b/arch/x86/kernel/amd_gart_64.c
@@ -180,22 +180,23 @@ static void iommu_full(struct device *dev, size_t size, int dir)
}
static inline int
-need_iommu(struct device *dev, unsigned long addr, size_t size)
+need_iommu(struct device *dev, unsigned long addr, size_t size, unsigned long attrs)
{
- return force_iommu || !dma_capable(dev, addr, size, true);
+ return force_iommu || !dma_capable(dev, addr, size, true, attrs);
}
static inline int
-nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
+nonforced_iommu(struct device *dev, unsigned long addr, size_t size,
+ unsigned long attrs)
{
- return !dma_capable(dev, addr, size, true);
+ return !dma_capable(dev, addr, size, true, attrs);
}
/* Map a single continuous physical area into the IOMMU.
* Caller needs to check if the iommu is needed and flush.
*/
static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
- size_t size, int dir, unsigned long align_mask)
+ size_t size, int dir, unsigned long align_mask, unsigned long attrs)
{
unsigned long npages = iommu_num_pages(phys_mem, size, PAGE_SIZE);
unsigned long iommu_page;
@@ -206,7 +207,7 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
iommu_page = alloc_iommu(dev, npages, align_mask);
if (iommu_page == -1) {
- if (!nonforced_iommu(dev, phys_mem, size))
+ if (!nonforced_iommu(dev, phys_mem, size, attrs))
return phys_mem;
if (panic_on_overflow)
panic("dma_map_area overflow %lu bytes\n", size);
@@ -231,10 +232,10 @@ static dma_addr_t gart_map_phys(struct device *dev, phys_addr_t paddr,
if (unlikely(attrs & DMA_ATTR_MMIO))
return DMA_MAPPING_ERROR;
- if (!need_iommu(dev, paddr, size))
+ if (!need_iommu(dev, paddr, size, attrs))
return paddr;
- bus = dma_map_area(dev, paddr, size, dir, 0);
+ bus = dma_map_area(dev, paddr, size, dir, 0, attrs);
flush_gart();
return bus;
@@ -289,7 +290,7 @@ static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
/* Fallback for dma_map_sg in case of overflow */
static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
- int nents, int dir)
+ int nents, int dir, unsigned long attrs)
{
struct scatterlist *s;
int i;
@@ -301,8 +302,8 @@ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
for_each_sg(sg, s, nents, i) {
unsigned long addr = sg_phys(s);
- if (nonforced_iommu(dev, addr, s->length)) {
- addr = dma_map_area(dev, addr, s->length, dir, 0);
+ if (nonforced_iommu(dev, addr, s->length, attrs)) {
+ addr = dma_map_area(dev, addr, s->length, dir, 0, attrs);
if (addr == DMA_MAPPING_ERROR) {
if (i > 0)
gart_unmap_sg(dev, sg, i, dir, 0);
@@ -401,7 +402,7 @@ static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
s->dma_address = addr;
BUG_ON(s->length == 0);
- nextneed = need_iommu(dev, addr, s->length);
+ nextneed = need_iommu(dev, addr, s->length, attrs);
/* Handle the previous not yet processed entries */
if (i > start) {
@@ -449,7 +450,7 @@ static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents,
/* When it was forced or merged try again in a dumb way */
if (force_iommu || iommu_merge) {
- out = dma_map_sg_nonforce(dev, sg, nents, dir);
+ out = dma_map_sg_nonforce(dev, sg, nents, dir, attrs);
if (out > 0)
return out;
}
@@ -473,7 +474,8 @@ gart_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr,
return vaddr;
*dma_addr = dma_map_area(dev, virt_to_phys(vaddr), size,
- DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);
+ DMA_BIDIRECTIONAL,
+ (1UL << get_order(size)) - 1, attrs);
flush_gart();
if (unlikely(*dma_addr == DMA_MAPPING_ERROR))
goto out_free;
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 8c4abe65cd49..e2538824ef52 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -212,7 +212,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
BUG_ON(dir == DMA_NONE);
if (attrs & DMA_ATTR_MMIO) {
- if (unlikely(!dma_capable(dev, phys, size, false))) {
+ if (unlikely(!dma_capable(dev, phys, size, false, attrs))) {
dev_err_once(
dev,
"DMA addr %pa+%zu overflow (mask %llx, bus limit %llx).\n",
@@ -231,7 +231,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
* we can safely return the device addr and not worry about bounce
* buffering it.
*/
- if (dma_capable(dev, dev_addr, size, true) &&
+ if (dma_capable(dev, dev_addr, size, true, attrs) &&
!dma_kmalloc_needs_bounce(dev, size, dir) &&
!range_straddles_page_boundary(phys, size) &&
!xen_arch_need_swiotlb(dev, phys, dev_addr) &&
@@ -253,7 +253,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
/*
* Ensure that the address returned is DMA'ble
*/
- if (unlikely(!dma_capable(dev, dev_addr, size, true))) {
+ if (unlikely(!dma_capable(dev, dev_addr, size, true, attrs))) {
__swiotlb_tbl_unmap_single(dev, map, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC,
swiotlb_find_pool(dev, map));
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index 94fad4e7c11e..daa31a1adf7b 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -135,12 +135,20 @@ static inline bool force_dma_unencrypted(struct device *dev)
#endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */
static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size,
- bool is_ram)
+ bool is_ram, unsigned long attrs)
{
dma_addr_t end = addr + size - 1;
if (addr == DMA_MAPPING_ERROR)
return false;
+ /*
+ * The DMA address was derived from encrypted RAM, but this device
+ * requires unencrypted DMA addresses. Treat it as not DMA-capable
+ * so the caller can fall back to a suitable SWIOTLB pool.
+ */
+ if (!(attrs & DMA_ATTR_CC_SHARED) && force_dma_unencrypted(dev))
+ return false;
+
if (is_ram && !IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) &&
min(addr, end) < phys_to_dma(dev, PFN_PHYS(min_low_pfn)))
return false;
diff --git a/kernel/dma/direct.h b/kernel/dma/direct.h
index 7140c208c123..e05dc7649366 100644
--- a/kernel/dma/direct.h
+++ b/kernel/dma/direct.h
@@ -101,15 +101,15 @@ static inline dma_addr_t dma_direct_map_phys(struct device *dev,
if (attrs & DMA_ATTR_MMIO) {
dma_addr = phys;
- if (unlikely(!dma_capable(dev, dma_addr, size, false)))
+ if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs)))
goto err_overflow;
} else if (attrs & DMA_ATTR_CC_SHARED) {
dma_addr = phys_to_dma_unencrypted(dev, phys);
- if (unlikely(!dma_capable(dev, dma_addr, size, false)))
+ if (unlikely(!dma_capable(dev, dma_addr, size, false, attrs)))
goto err_overflow;
} else {
dma_addr = phys_to_dma(dev, phys);
- if (unlikely(!dma_capable(dev, dma_addr, size, true)) ||
+ if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs)) ||
dma_kmalloc_needs_bounce(dev, size, dir)) {
if (is_swiotlb_active(dev) &&
!(attrs & DMA_ATTR_REQUIRE_COHERENT))
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 335e27bc1e1f..b5960c2e98d8 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1697,7 +1697,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
else
dma_addr = phys_to_dma_encrypted(dev, swiotlb_addr);
- if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
+ if (unlikely(!dma_capable(dev, dma_addr, size, true, attrs))) {
__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC,
swiotlb_find_pool(dev, swiotlb_addr));
--
2.43.0
^ permalink raw reply related
* [PATCH v7 14/22] dma-mapping: make dma_pgprot() honor __DMA_ATTR_ALLOC_CC_SHARED
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko,
Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
Fold encrypted/decrypted pgprot selection into dma_pgprot() so callers
do not need to adjust the page protection separately.
Update dma_pgprot() to apply pgprot_decrypted() when DMA_ATTR_CC_SHARED or
__DMA_ATTR_ALLOC_CC_SHARED is set and pgprot_encrypted() otherwise Convert
the dma-direct mmap paths to pass DMA_ATTR_CC_SHARED instead of open-coding
force_dma_unencrypted() handling around dma_pgprot().
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 8 +++-----
kernel/dma/mapping.c | 16 ++++++++++++----
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 64cb9f13ef03..9935f7caa459 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -283,9 +283,6 @@ void *dma_direct_alloc(struct device *dev, size_t size,
if (remap) {
pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
- if (force_dma_unencrypted(dev))
- prot = pgprot_decrypted(prot);
-
/* remove any dirty cache lines on the kernel alias */
arch_dma_prep_coherent(page, size);
@@ -605,9 +602,10 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr));
int ret = -ENXIO;
- vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
if (force_dma_unencrypted(dev))
- vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
+ attrs |= DMA_ATTR_CC_SHARED;
+
+ vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
return ret;
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index d2f70b6ccd0f..a628820fd10e 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -537,13 +537,21 @@ EXPORT_SYMBOL(dma_get_sgtable_attrs);
*/
pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs)
{
+ pgprot_t dma_prot;
+
if (dev_is_dma_coherent(dev))
- return prot;
+ dma_prot = prot;
#ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE
- if (attrs & DMA_ATTR_WRITE_COMBINE)
- return pgprot_writecombine(prot);
+ else if (attrs & DMA_ATTR_WRITE_COMBINE)
+ dma_prot = pgprot_writecombine(prot);
#endif
- return pgprot_dmacoherent(prot);
+ else
+ dma_prot = pgprot_dmacoherent(prot);
+
+ if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED))
+ return pgprot_decrypted(dma_prot);
+ else
+ return pgprot_encrypted(dma_prot);
}
#endif /* CONFIG_MMU */
--
2.43.0
^ permalink raw reply related
* [PATCH v7 13/22] dma: swiotlb: track pool encryption state and honor DMA_ATTR_CC_SHARED
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko,
Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
Teach swiotlb to distinguish between encrypted and decrypted bounce
buffer pools, and make allocation and mapping paths select a pool whose
state matches the requested DMA attributes.
Add a cc_shared flag to io_tlb_mem, initialize it for the default and
restricted pools, and propagate __DMA_ATTR_ALLOC_CC_SHARED into swiotlb
pool allocation. Reject swiotlb alloc/map requests when the selected pool
does not match the required encrypted/decrypted state.
Also return DMA addresses with the matching phys_to_dma_{encrypted,
unencrypted} helper so the DMA address encoding stays consistent with the
chosen pool.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
include/linux/dma-direct.h | 10 +++
include/linux/swiotlb.h | 10 ++-
kernel/dma/direct.c | 13 ++-
kernel/dma/swiotlb.c | 177 ++++++++++++++++++++++++++++---------
4 files changed, 162 insertions(+), 48 deletions(-)
diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index c249912456f9..94fad4e7c11e 100644
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -77,6 +77,10 @@ static inline dma_addr_t dma_range_map_max(const struct bus_dma_region *map)
#ifndef phys_to_dma_unencrypted
#define phys_to_dma_unencrypted phys_to_dma
#endif
+
+#ifndef phys_to_dma_encrypted
+#define phys_to_dma_encrypted phys_to_dma
+#endif
#else
static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr)
{
@@ -90,6 +94,12 @@ static inline dma_addr_t phys_to_dma_unencrypted(struct device *dev,
{
return dma_addr_unencrypted(__phys_to_dma(dev, paddr));
}
+
+static inline dma_addr_t phys_to_dma_encrypted(struct device *dev,
+ phys_addr_t paddr)
+{
+ return dma_addr_encrypted(__phys_to_dma(dev, paddr));
+}
/*
* If memory encryption is supported, phys_to_dma will set the memory encryption
* bit in the DMA address, and dma_to_phys will clear it.
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index ea4c0a292dea..ee42f7588847 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -66,6 +66,7 @@ extern void __init swiotlb_update_mem_attributes(void);
* @node: Member of the IO TLB memory pool list.
* @rcu: RCU head for swiotlb_dyn_free().
* @transient: %true if transient memory pool.
+ * @cc_shared: %true if the pool memory is shared for confidential computing.
*/
struct io_tlb_pool {
phys_addr_t start;
@@ -81,6 +82,7 @@ struct io_tlb_pool {
struct list_head node;
struct rcu_head rcu;
bool transient;
+ bool cc_shared;
#endif
};
@@ -92,6 +94,7 @@ struct io_tlb_pool {
* @debugfs: The dentry to debugfs.
* @force_bounce: %true if swiotlb bouncing is forced
* @for_alloc: %true if the pool is used for memory allocation
+ * @cc_shared: %true if the pool memory is shared for confidential computing.
* @can_grow: %true if more pools can be allocated dynamically.
* @phys_limit: Maximum allowed physical address.
* @lock: Lock to synchronize changes to the list.
@@ -111,6 +114,7 @@ struct io_tlb_mem {
struct dentry *debugfs;
bool force_bounce;
bool for_alloc;
+ bool cc_shared;
#ifdef CONFIG_SWIOTLB_DYNAMIC
bool can_grow;
u64 phys_limit;
@@ -282,7 +286,8 @@ static inline void swiotlb_sync_single_for_cpu(struct device *dev,
extern void swiotlb_print_info(void);
#ifdef CONFIG_DMA_RESTRICTED_POOL
-struct page *swiotlb_alloc(struct device *dev, size_t size);
+struct page *swiotlb_alloc(struct device *dev, size_t size,
+ unsigned long attrs);
bool swiotlb_free(struct device *dev, struct page *page, size_t size);
void swiotlb_free_from_pool(struct device *dev,
phys_addr_t tlb_addr, struct io_tlb_pool *pool);
@@ -292,7 +297,8 @@ static inline bool is_swiotlb_for_alloc(struct device *dev)
return dev->dma_io_tlb_mem->for_alloc;
}
#else
-static inline struct page *swiotlb_alloc(struct device *dev, size_t size)
+static inline struct page *swiotlb_alloc(struct device *dev, size_t size,
+ unsigned long attrs)
{
return NULL;
}
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index c50d6987438e..64cb9f13ef03 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -96,9 +96,10 @@ static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)
return ret;
}
-static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)
+static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size,
+ unsigned long attrs)
{
- struct page *page = swiotlb_alloc(dev, size);
+ struct page *page = swiotlb_alloc(dev, size, attrs);
if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {
swiotlb_free(dev, page, size);
@@ -251,8 +252,12 @@ void *dma_direct_alloc(struct device *dev, size_t size,
}
if (is_swiotlb_for_alloc(dev)) {
- page = dma_direct_alloc_swiotlb(dev, size);
+ page = dma_direct_alloc_swiotlb(dev, size, attrs);
if (page) {
+ /*
+ * swiotlb allocations comes from pool already marked
+ * decrypted
+ */
mark_mem_decrypt = false;
goto setup_page;
}
@@ -398,7 +403,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
&ret, gfp, attrs);
if (is_swiotlb_for_alloc(dev)) {
- page = dma_direct_alloc_swiotlb(dev, size);
+ page = dma_direct_alloc_swiotlb(dev, size, attrs);
if (!page)
return NULL;
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 046ae92c4832..335e27bc1e1f 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -259,10 +259,21 @@ void __init swiotlb_update_mem_attributes(void)
struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
unsigned long bytes;
+ /*
+ * if platform support memory encryption, swiotlb buffers are
+ * shared by default.
+ */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+ io_tlb_default_mem.cc_shared = true;
+ else
+ io_tlb_default_mem.cc_shared = false;
+
if (!mem->nslabs || mem->late_alloc)
return;
bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
- set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
+
+ if (io_tlb_default_mem.cc_shared)
+ set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT);
}
static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
@@ -505,8 +516,10 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
if (!mem->slots)
goto error_slots;
- set_memory_decrypted((unsigned long)vstart,
- (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+ if (io_tlb_default_mem.cc_shared)
+ set_memory_decrypted((unsigned long)vstart,
+ (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
+
swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), vstart, nslabs, true,
nareas);
add_mem_pool(&io_tlb_default_mem, mem);
@@ -539,7 +552,9 @@ void __init swiotlb_exit(void)
tbl_size = PAGE_ALIGN(mem->end - mem->start);
slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
- set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+ if (io_tlb_default_mem.cc_shared)
+ set_memory_encrypted(tbl_vaddr, tbl_size >> PAGE_SHIFT);
+
if (mem->late_alloc) {
area_order = get_order(array_size(sizeof(*mem->areas),
mem->nareas));
@@ -563,6 +578,7 @@ void __init swiotlb_exit(void)
* @gfp: GFP flags for the allocation.
* @bytes: Size of the buffer.
* @phys_limit: Maximum allowed physical address of the buffer.
+ * @attrs: DMA attributes for the allocation.
*
* Allocate pages from the buddy allocator. If successful, make the allocated
* pages decrypted that they can be used for DMA.
@@ -570,9 +586,11 @@ void __init swiotlb_exit(void)
* Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
* if the allocated physical address was above @phys_limit.
*/
-static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
+static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes,
+ u64 phys_limit, unsigned long attrs)
{
unsigned int order = get_order(bytes);
+ bool cc_shared = attrs & __DMA_ATTR_ALLOC_CC_SHARED;
struct page *page;
phys_addr_t paddr;
void *vaddr;
@@ -588,13 +606,13 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
}
vaddr = phys_to_virt(paddr);
- if (set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
+ if (cc_shared && set_memory_decrypted((unsigned long)vaddr, PFN_UP(bytes)))
goto error;
return page;
error:
/* Intentional leak if pages cannot be encrypted again. */
- if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
+ if (cc_shared && !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
__free_pages(page, order);
return NULL;
}
@@ -602,6 +620,7 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
/**
* swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
* @dev: Device for which a memory pool is allocated.
+ * @mem: SWIOTLB allocator for the pool.
* @bytes: Size of the buffer.
* @phys_limit: Maximum allowed physical address of the buffer.
* @gfp: GFP flags for the allocation.
@@ -609,25 +628,23 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit)
*
* Return: Allocated pages, or %NULL on allocation failure.
*/
-static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
+static struct page *swiotlb_alloc_tlb(struct device *dev,
+ struct io_tlb_mem *mem, size_t bytes,
u64 phys_limit, gfp_t gfp, void **vaddr)
{
struct page *page;
- unsigned long attrs = 0;
+ unsigned long attrs = mem->cc_shared ? __DMA_ATTR_ALLOC_CC_SHARED : 0;
*vaddr = NULL;
/*
* Allocate from the atomic pools if memory is encrypted and
* the allocation is atomic, because decrypting may block.
*/
- if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) {
+ if (!gfpflags_allow_blocking(gfp) && dev && mem->cc_shared) {
+
if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
return NULL;
- /* swiotlb considered decrypted by default */
- if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
- attrs = __DMA_ATTR_ALLOC_CC_SHARED;
-
return dma_alloc_from_pool(dev, bytes, vaddr, gfp,
attrs, dma_coherent_ok);
}
@@ -638,7 +655,7 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
else if (phys_limit <= DMA_BIT_MASK(32))
gfp |= __GFP_DMA32;
- while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit))) {
+ while (IS_ERR(page = alloc_dma_pages(gfp, bytes, phys_limit, attrs))) {
if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
phys_limit < DMA_BIT_MASK(64) &&
!(gfp & (__GFP_DMA32 | __GFP_DMA)))
@@ -659,21 +676,25 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
* swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
* @vaddr: Virtual address of the buffer.
* @bytes: Size of the buffer.
+ * @cc_shared: true if @vaddr was allocated decrypted and must be
+ * re-encrypted before being freed
*/
-static void swiotlb_free_tlb(void *vaddr, size_t bytes)
+static void swiotlb_free_tlb(void *vaddr, size_t bytes, bool cc_shared)
{
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
dma_free_from_pool(NULL, vaddr, bytes))
return;
/* Intentional leak if pages cannot be encrypted again. */
- if (!set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
+ if (!cc_shared ||
+ !set_memory_encrypted((unsigned long)vaddr, PFN_UP(bytes)))
__free_pages(virt_to_page(vaddr), get_order(bytes));
}
/**
* swiotlb_alloc_pool() - allocate a new IO TLB memory pool
* @dev: Device for which a memory pool is allocated.
+ * @mem: SWIOTLB allocator for the pool.
* @minslabs: Minimum number of slabs.
* @nslabs: Desired (maximum) number of slabs.
* @nareas: Number of areas.
@@ -687,8 +708,9 @@ static void swiotlb_free_tlb(void *vaddr, size_t bytes)
* Return: New memory pool, or %NULL on allocation failure.
*/
static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
- unsigned long minslabs, unsigned long nslabs,
- unsigned int nareas, u64 phys_limit, gfp_t gfp)
+ struct io_tlb_mem *mem, unsigned long minslabs,
+ unsigned long nslabs, unsigned int nareas, u64 phys_limit,
+ gfp_t gfp)
{
struct io_tlb_pool *pool;
unsigned int slot_order;
@@ -707,10 +729,11 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
if (!pool)
goto error;
pool->areas = (void *)pool + sizeof(*pool);
+ pool->cc_shared = mem->cc_shared;
tlb_size = nslabs << IO_TLB_SHIFT;
- while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp,
- &tlb_vaddr))) {
+ while (!(tlb = swiotlb_alloc_tlb(dev, mem, tlb_size,
+ phys_limit, gfp, &tlb_vaddr))) {
if (nslabs <= minslabs)
goto error_tlb;
nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
@@ -729,7 +752,7 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
return pool;
error_slots:
- swiotlb_free_tlb(tlb_vaddr, tlb_size);
+ swiotlb_free_tlb(tlb_vaddr, tlb_size, mem->cc_shared);
error_tlb:
kfree(pool);
error:
@@ -746,7 +769,7 @@ static void swiotlb_dyn_alloc(struct work_struct *work)
container_of(work, struct io_tlb_mem, dyn_alloc);
struct io_tlb_pool *pool;
- pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,
+ pool = swiotlb_alloc_pool(NULL, mem, IO_TLB_MIN_SLABS, default_nslabs,
default_nareas, mem->phys_limit, GFP_KERNEL);
if (!pool) {
pr_warn_ratelimited("Failed to allocate new pool");
@@ -767,7 +790,7 @@ static void swiotlb_dyn_free(struct rcu_head *rcu)
size_t tlb_size = pool->end - pool->start;
free_pages((unsigned long)pool->slots, get_order(slots_size));
- swiotlb_free_tlb(pool->vaddr, tlb_size);
+ swiotlb_free_tlb(pool->vaddr, tlb_size, pool->cc_shared);
kfree(pool);
}
@@ -1031,6 +1054,7 @@ static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
* @pool: Memory pool to be searched.
* @area_index: Index of the IO TLB memory area to be searched.
* @orig_addr: Original (non-bounced) IO buffer address.
+ * @tbl_dma_addr: DMA address of the bounce buffer.
* @alloc_size: Total requested size of the bounce buffer,
* including initial alignment padding.
* @alloc_align_mask: Required alignment of the allocated buffer.
@@ -1042,13 +1066,11 @@ static void dec_transient_used(struct io_tlb_mem *mem, unsigned int nslots)
* Return: Index of the first allocated slot, or -1 on error.
*/
static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,
- int area_index, phys_addr_t orig_addr, size_t alloc_size,
- unsigned int alloc_align_mask)
+ int area_index, phys_addr_t orig_addr, dma_addr_t tbl_dma_addr,
+ size_t alloc_size, unsigned int alloc_align_mask)
{
struct io_tlb_area *area = pool->areas + area_index;
unsigned long boundary_mask = dma_get_seg_boundary(dev);
- dma_addr_t tbl_dma_addr =
- phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
unsigned long max_slots = get_max_slots(boundary_mask);
unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
unsigned int nslots = nr_slots(alloc_size), stride;
@@ -1061,6 +1083,8 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
BUG_ON(!nslots);
BUG_ON(area_index >= pool->nareas);
+ tbl_dma_addr &= boundary_mask;
+
/*
* Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
* page-aligned in the absence of any other alignment requirements.
@@ -1172,6 +1196,7 @@ static int swiotlb_search_area(struct device *dev, int start_cpu,
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
+ dma_addr_t tbl_dma_addr;
int area_index;
int index = -1;
@@ -1180,9 +1205,15 @@ static int swiotlb_search_area(struct device *dev, int start_cpu,
if (cpu_offset >= pool->nareas)
continue;
area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);
+
+ if (mem->cc_shared)
+ tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
+ else
+ tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
+
index = swiotlb_search_pool_area(dev, pool, area_index,
- orig_addr, alloc_size,
- alloc_align_mask);
+ orig_addr, tbl_dma_addr,
+ alloc_size, alloc_align_mask);
if (index >= 0) {
*retpool = pool;
break;
@@ -1212,6 +1243,7 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
+ dma_addr_t tbl_dma_addr;
unsigned long nslabs;
unsigned long flags;
u64 phys_limit;
@@ -1236,12 +1268,17 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
nslabs = nr_slots(alloc_size);
phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
- pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,
+ pool = swiotlb_alloc_pool(dev, mem, nslabs, nslabs, 1, phys_limit,
GFP_NOWAIT);
if (!pool)
return -1;
- index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,
+ if (mem->cc_shared)
+ tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
+ else
+ tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
+
+ index = swiotlb_search_pool_area(dev, pool, 0, orig_addr, tbl_dma_addr,
alloc_size, alloc_align_mask);
if (index < 0) {
swiotlb_dyn_free(&pool->rcu);
@@ -1286,15 +1323,23 @@ static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
size_t alloc_size, unsigned int alloc_align_mask,
struct io_tlb_pool **retpool)
{
+ struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
+ dma_addr_t tbl_dma_addr;
int start, i;
int index;
- *retpool = pool = &dev->dma_io_tlb_mem->defpool;
+ *retpool = pool = &mem->defpool;
+ if (mem->cc_shared)
+ tbl_dma_addr = phys_to_dma_unencrypted(dev, pool->start);
+ else
+ tbl_dma_addr = phys_to_dma_encrypted(dev, pool->start);
+
i = start = raw_smp_processor_id() & (pool->nareas - 1);
do {
index = swiotlb_search_pool_area(dev, pool, i, orig_addr,
- alloc_size, alloc_align_mask);
+ tbl_dma_addr, alloc_size,
+ alloc_align_mask);
if (index >= 0)
return index;
if (++i >= pool->nareas)
@@ -1377,9 +1422,19 @@ static unsigned long mem_used(struct io_tlb_mem *mem)
* any pre- or post-padding for alignment
* @alloc_align_mask: Required start and end alignment of the allocated buffer
* @dir: DMA direction
- * @attrs: Optional DMA attributes for the map operation
+ * @attrs: Optional DMA attributes for the map operation, updated
+ * to match the selected SWIOTLB pool
*
* Find and allocate a suitable sequence of IO TLB slots for the request.
+ * The device's SWIOTLB pool must match the device's current DMA encryption
+ * requirements. If the device requires decrypted DMA, bouncing is done through
+ * an unencrypted pool and the mapping is marked shared. If the device can DMA
+ * to encrypted memory, bouncing is done through an encrypted pool even when the
+ * original DMA address was unencrypted. Enabling encrypted DMA for a device is
+ * therefore expected to update its default io_tlb_mem to an encrypted pool, so
+ * later bounce mappings for both encrypted and decrypted original memory use
+ * that encrypted pool.
+ *
* The allocated space starts at an alignment specified by alloc_align_mask,
* and the size of the allocated space is rounded up so that the total amount
* of allocated space is a multiple of (alloc_align_mask + 1). If
@@ -1416,6 +1471,30 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
+ if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) {
+
+ /* swiotlb pool is incorrect for this device */
+ if (unlikely(mem->cc_shared != force_dma_unencrypted(dev)))
+ return (phys_addr_t)DMA_MAPPING_ERROR;
+
+ } else if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) {
+ /*
+ * On hosts with memory encryption, SWIOTLB-backed memory is
+ * unencrypted. DMA addresses returned for bounce buffers must
+ * therefore be marked unencrypted, even for devices that can
+ * address encrypted memory. This also preserves swiotlb=force
+ * behavior for those devices.
+ */
+ if (unlikely(!mem->cc_shared))
+ return (phys_addr_t)DMA_MAPPING_ERROR;
+ }
+
+ /* Force attrs to match the kind of memory in the pool */
+ if (mem->cc_shared)
+ *attrs |= DMA_ATTR_CC_SHARED;
+ else
+ *attrs &= ~DMA_ATTR_CC_SHARED;
+
/*
* The default swiotlb memory pool is allocated with PAGE_SIZE
* alignment. If a mapping is requested with larger alignment,
@@ -1613,8 +1692,11 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
- /* Ensure that the address returned is DMA'ble */
- dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
+ if (attrs & DMA_ATTR_CC_SHARED)
+ dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
+ else
+ dma_addr = phys_to_dma_encrypted(dev, swiotlb_addr);
+
if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
__swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, dir,
attrs | DMA_ATTR_SKIP_CPU_SYNC,
@@ -1778,7 +1860,7 @@ static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
#ifdef CONFIG_DMA_RESTRICTED_POOL
-struct page *swiotlb_alloc(struct device *dev, size_t size)
+struct page *swiotlb_alloc(struct device *dev, size_t size, unsigned long attrs)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
struct io_tlb_pool *pool;
@@ -1789,6 +1871,9 @@ struct page *swiotlb_alloc(struct device *dev, size_t size)
if (!mem)
return NULL;
+ if (mem->cc_shared != !!(attrs & __DMA_ATTR_ALLOC_CC_SHARED))
+ return NULL;
+
align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
index = swiotlb_find_slots(dev, 0, size, align, &pool);
if (index == -1)
@@ -1864,12 +1949,20 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
kfree(mem);
return -ENOMEM;
}
+ /*
+ * if platform supports memory encryption,
+ * restricted mem pool is shared by default
+ */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) {
+ mem->cc_shared = true;
+ set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
+ rmem->size >> PAGE_SHIFT);
+ } else {
+ mem->cc_shared = false;
+ }
- set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
- rmem->size >> PAGE_SHIFT);
swiotlb_init_io_tlb_pool(pool, rmem->base, phys_to_virt(rmem->base),
- nslabs,
- false, nareas);
+ nslabs, false, nareas);
mem->force_bounce = true;
mem->for_alloc = true;
#ifdef CONFIG_SWIOTLB_DYNAMIC
--
2.43.0
^ permalink raw reply related
* [PATCH v7 12/22] dma: swiotlb: pass mapping attributes by reference
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
Change swiotlb_tbl_map_single() to take the DMA mapping attributes by
reference and update the direct callers accordingly.
This is a preparatory change for a follow-up patch which updates the
attributes based on the selected swiotlb pool. Keeping the signature change
separate makes the follow-up patch easier to review.
No functional change in this patch.
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
drivers/iommu/dma-iommu.c | 2 +-
drivers/xen/swiotlb-xen.c | 2 +-
include/linux/swiotlb.h | 2 +-
kernel/dma/swiotlb.c | 6 +++---
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index a7b1da5e06e6..fe387829ee92 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1180,7 +1180,7 @@ static phys_addr_t iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys,
trace_swiotlb_bounced(dev, phys, size);
phys = swiotlb_tbl_map_single(dev, phys, size, iova_mask(iovad), dir,
- attrs);
+ &attrs);
/*
* Untrusted devices should not see padding areas with random leftover
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index 2cbf2b588f5b..8c4abe65cd49 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -243,7 +243,7 @@ static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys,
*/
trace_swiotlb_bounced(dev, dev_addr, size);
- map = swiotlb_tbl_map_single(dev, phys, size, 0, dir, attrs);
+ map = swiotlb_tbl_map_single(dev, phys, size, 0, dir, &attrs);
if (map == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index c92ff6791595..ea4c0a292dea 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -238,7 +238,7 @@ static inline phys_addr_t default_swiotlb_limit(void)
phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
size_t mapping_size, unsigned int alloc_aligned_mask,
- enum dma_data_direction dir, unsigned long attrs);
+ enum dma_data_direction dir, unsigned long *attrs);
dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
size_t size, enum dma_data_direction dir, unsigned long attrs);
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 908de28aceb2..046ae92c4832 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1396,7 +1396,7 @@ static unsigned long mem_used(struct io_tlb_mem *mem)
*/
phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
size_t mapping_size, unsigned int alloc_align_mask,
- enum dma_data_direction dir, unsigned long attrs)
+ enum dma_data_direction dir, unsigned long *attrs)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
unsigned int offset;
@@ -1430,7 +1430,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
size = ALIGN(mapping_size + offset, alloc_align_mask + 1);
index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool);
if (index == -1) {
- if (!(attrs & DMA_ATTR_NO_WARN))
+ if (!(*attrs & DMA_ATTR_NO_WARN))
dev_warn_ratelimited(dev,
"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
size, mem->nslabs, mem_used(mem));
@@ -1609,7 +1609,7 @@ dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size);
- swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, attrs);
+ swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, 0, dir, &attrs);
if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
return DMA_MAPPING_ERROR;
--
2.43.0
^ permalink raw reply related
* [PATCH v7 11/22] dma-pool: track decrypted atomic pools and select them via attrs
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko,
Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
Teach the atomic DMA pool code to distinguish between encrypted and
unencrypted pools, and make pool allocation select the matching pool based
on DMA attributes.
Introduce a dma_gen_pool wrapper that records whether a pool is
unencrypted, initialize that state when the atomic pools are created, and
use it when expanding and resizing the pools. Update dma_alloc_from_pool()
to take attrs and skip pools whose encrypted state does not match
__DMA_ATTR_ALLOC_CC_SHARED. Update dma_free_from_pool() accordingly.
Also pass __DMA_ATTR_ALLOC_CC_SHARED from the swiotlb atomic allocation
path so decrypted swiotlb allocations are taken from the correct atomic
pool.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
drivers/iommu/dma-iommu.c | 7 +-
include/linux/dma-map-ops.h | 2 +-
kernel/dma/direct.c | 11 ++-
kernel/dma/pool.c | 173 +++++++++++++++++++++++-------------
kernel/dma/swiotlb.c | 8 +-
5 files changed, 129 insertions(+), 72 deletions(-)
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 68c686c1e81a..a7b1da5e06e6 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -1660,9 +1660,14 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
{
bool coherent = dev_is_dma_coherent(dev);
int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
+ bool is_alloc_cc_shared = attrs & __DMA_ATTR_ALLOC_CC_SHARED;
struct page *page = NULL;
void *cpu_addr;
+ /* Not yet supported */
+ if (is_alloc_cc_shared)
+ return NULL;
+
gfp |= __GFP_ZERO;
if (gfpflags_allow_blocking(gfp) &&
@@ -1673,7 +1678,7 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
!gfpflags_allow_blocking(gfp) && !coherent) {
page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
- gfp, NULL);
+ gfp, attrs, NULL);
if (!page)
return NULL;
} else {
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index 137e015c1750..8fae2b7deb20 100644
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -212,7 +212,7 @@ void *dma_common_pages_remap(struct page **pages, size_t size, pgprot_t prot,
void dma_common_free_remap(void *cpu_addr, size_t size);
struct page *dma_alloc_from_pool(struct device *dev, size_t size,
- void **cpu_addr, gfp_t flags,
+ void **cpu_addr, gfp_t flags, unsigned long attrs,
bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t));
bool dma_free_from_pool(struct device *dev, void *start, size_t size);
bool dma_free_from_pool_page(struct device *dev, struct page *page, size_t size);
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 98e47e0b332d..c50d6987438e 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -154,7 +154,8 @@ static bool dma_direct_use_pool(struct device *dev, gfp_t gfp)
}
static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size,
- dma_addr_t *dma_handle, void **cpu_addr, gfp_t gfp)
+ dma_addr_t *dma_handle, void **cpu_addr, gfp_t gfp,
+ unsigned long attrs)
{
struct page *page;
u64 phys_limit;
@@ -163,7 +164,8 @@ static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size,
return NULL;
gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit);
- page = dma_alloc_from_pool(dev, size, cpu_addr, gfp, dma_coherent_ok);
+ page = dma_alloc_from_pool(dev, size, cpu_addr, gfp, attrs,
+ dma_coherent_ok);
if (!page)
return NULL;
*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));
@@ -244,7 +246,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
if ((remap || (attrs & __DMA_ATTR_ALLOC_CC_SHARED)) &&
dma_direct_use_pool(dev, gfp)) {
page = dma_direct_alloc_from_pool(dev, size, dma_handle,
- &ret, gfp);
+ &ret, gfp, attrs);
return page ? ret : NULL;
}
@@ -392,7 +394,8 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp))
- return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp);
+ return dma_direct_alloc_from_pool(dev, size, dma_handle,
+ &ret, gfp, attrs);
if (is_swiotlb_for_alloc(dev)) {
page = dma_direct_alloc_swiotlb(dev, size);
diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c
index 76bcafe03e44..5115a3e8c722 100644
--- a/kernel/dma/pool.c
+++ b/kernel/dma/pool.c
@@ -12,12 +12,18 @@
#include <linux/set_memory.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
+#include <linux/cc_platform.h>
-static struct gen_pool *atomic_pool_dma __ro_after_init;
+struct dma_gen_pool {
+ bool cc_shared;
+ struct gen_pool *pool;
+};
+
+static struct dma_gen_pool atomic_pool_dma __ro_after_init;
static unsigned long pool_size_dma;
-static struct gen_pool *atomic_pool_dma32 __ro_after_init;
+static struct dma_gen_pool atomic_pool_dma32 __ro_after_init;
static unsigned long pool_size_dma32;
-static struct gen_pool *atomic_pool_kernel __ro_after_init;
+static struct dma_gen_pool atomic_pool_kernel __ro_after_init;
static unsigned long pool_size_kernel;
/* Size can be defined by the coherent_pool command line */
@@ -76,7 +82,7 @@ static bool cma_in_zone(gfp_t gfp)
return true;
}
-static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
+static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_size,
gfp_t gfp)
{
unsigned int order;
@@ -114,14 +120,17 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
* Memory in the atomic DMA pools must be unencrypted, the pools do not
* shrink so no re-encryption occurs in dma_direct_free().
*/
- ret = set_memory_decrypted((unsigned long)page_to_virt(page),
- 1 << order);
- if (ret) {
- leak_pages = true;
- goto remove_mapping;
+ if (dma_pool->cc_shared) {
+ ret = set_memory_decrypted((unsigned long)page_to_virt(page),
+ 1 << order);
+ if (ret) {
+ leak_pages = true;
+ goto remove_mapping;
+ }
}
- ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),
- pool_size, NUMA_NO_NODE);
+
+ ret = gen_pool_add_virt(dma_pool->pool, (unsigned long)addr,
+ page_to_phys(page), pool_size, NUMA_NO_NODE);
if (ret)
goto encrypt_mapping;
@@ -129,12 +138,10 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
return 0;
encrypt_mapping:
- ret = set_memory_encrypted((unsigned long)page_to_virt(page),
- 1 << order);
- if (WARN_ON_ONCE(ret)) {
- /* Decrypt succeeded but encrypt failed, purposely leak */
+ if (dma_pool->cc_shared &&
+ set_memory_encrypted((unsigned long)page_to_virt(page), 1 << order))
leak_pages = true;
- }
+
remove_mapping:
#ifdef CONFIG_DMA_DIRECT_REMAP
dma_common_free_remap(addr, pool_size);
@@ -146,46 +153,52 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,
return ret;
}
-static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp)
+static void atomic_pool_resize(struct dma_gen_pool *dma_pool, gfp_t gfp)
{
- if (pool && gen_pool_avail(pool) < atomic_pool_size)
- atomic_pool_expand(pool, gen_pool_size(pool), gfp);
+ if (dma_pool->pool && gen_pool_avail(dma_pool->pool) < atomic_pool_size)
+ atomic_pool_expand(dma_pool, gen_pool_size(dma_pool->pool), gfp);
}
static void atomic_pool_work_fn(struct work_struct *work)
{
if (IS_ENABLED(CONFIG_ZONE_DMA))
- atomic_pool_resize(atomic_pool_dma,
+ atomic_pool_resize(&atomic_pool_dma,
GFP_KERNEL | GFP_DMA);
if (IS_ENABLED(CONFIG_ZONE_DMA32))
- atomic_pool_resize(atomic_pool_dma32,
+ atomic_pool_resize(&atomic_pool_dma32,
GFP_KERNEL | GFP_DMA32);
- atomic_pool_resize(atomic_pool_kernel, GFP_KERNEL);
+ atomic_pool_resize(&atomic_pool_kernel, GFP_KERNEL);
}
-static __init struct gen_pool *__dma_atomic_pool_init(size_t pool_size,
- gfp_t gfp)
+static __init struct dma_gen_pool *__dma_atomic_pool_init(struct dma_gen_pool *dma_pool,
+ size_t pool_size, gfp_t gfp)
{
- struct gen_pool *pool;
int ret;
- pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
- if (!pool)
+ dma_pool->pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
+ if (!dma_pool->pool)
return NULL;
- gen_pool_set_algo(pool, gen_pool_first_fit_order_align, NULL);
+ gen_pool_set_algo(dma_pool->pool, gen_pool_first_fit_order_align, NULL);
+
+ /* if platform is using memory encryption atomic pools are by default shared. */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+ dma_pool->cc_shared = true;
+ else
+ dma_pool->cc_shared = false;
- ret = atomic_pool_expand(pool, pool_size, gfp);
+ ret = atomic_pool_expand(dma_pool, pool_size, gfp);
if (ret) {
- gen_pool_destroy(pool);
+ gen_pool_destroy(dma_pool->pool);
+ dma_pool->pool = NULL;
pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",
pool_size >> 10, &gfp);
return NULL;
}
pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",
- gen_pool_size(pool) >> 10, &gfp);
- return pool;
+ gen_pool_size(dma_pool->pool) >> 10, &gfp);
+ return dma_pool;
}
#ifdef CONFIG_ZONE_DMA32
@@ -211,21 +224,22 @@ static int __init dma_atomic_pool_init(void)
/* All memory might be in the DMA zone(s) to begin with */
if (has_managed_zone(ZONE_NORMAL)) {
- atomic_pool_kernel = __dma_atomic_pool_init(atomic_pool_size,
- GFP_KERNEL);
- if (!atomic_pool_kernel)
+ __dma_atomic_pool_init(&atomic_pool_kernel, atomic_pool_size, GFP_KERNEL);
+ if (!atomic_pool_kernel.pool)
ret = -ENOMEM;
}
+
if (has_managed_dma()) {
- atomic_pool_dma = __dma_atomic_pool_init(atomic_pool_size,
- GFP_KERNEL | GFP_DMA);
- if (!atomic_pool_dma)
+ __dma_atomic_pool_init(&atomic_pool_dma, atomic_pool_size,
+ GFP_KERNEL | GFP_DMA);
+ if (!atomic_pool_dma.pool)
ret = -ENOMEM;
}
+
if (has_managed_dma32) {
- atomic_pool_dma32 = __dma_atomic_pool_init(atomic_pool_size,
- GFP_KERNEL | GFP_DMA32);
- if (!atomic_pool_dma32)
+ __dma_atomic_pool_init(&atomic_pool_dma32, atomic_pool_size,
+ GFP_KERNEL | GFP_DMA32);
+ if (!atomic_pool_dma32.pool)
ret = -ENOMEM;
}
@@ -234,19 +248,44 @@ static int __init dma_atomic_pool_init(void)
}
postcore_initcall(dma_atomic_pool_init);
-static inline struct gen_pool *dma_guess_pool(struct gen_pool *prev, gfp_t gfp)
+static inline struct dma_gen_pool *__dma_guess_pool(struct dma_gen_pool *first,
+ struct dma_gen_pool *second, struct dma_gen_pool *third)
{
- if (prev == NULL) {
+ if (first->pool)
+ return first;
+ if (second && second->pool)
+ return second;
+ if (third && third->pool)
+ return third;
+ return NULL;
+}
+
+static inline struct dma_gen_pool *dma_guess_pool(struct dma_gen_pool *prev,
+ gfp_t gfp)
+{
+ if (!prev) {
if (gfp & GFP_DMA)
- return atomic_pool_dma ?: atomic_pool_dma32 ?: atomic_pool_kernel;
+ return __dma_guess_pool(&atomic_pool_dma,
+ &atomic_pool_dma32,
+ &atomic_pool_kernel);
+
if (gfp & GFP_DMA32)
- return atomic_pool_dma32 ?: atomic_pool_dma ?: atomic_pool_kernel;
- return atomic_pool_kernel ?: atomic_pool_dma32 ?: atomic_pool_dma;
+ return __dma_guess_pool(&atomic_pool_dma32,
+ &atomic_pool_dma,
+ &atomic_pool_kernel);
+
+ return __dma_guess_pool(&atomic_pool_kernel,
+ &atomic_pool_dma32,
+ &atomic_pool_dma);
}
- if (prev == atomic_pool_kernel)
- return atomic_pool_dma32 ? atomic_pool_dma32 : atomic_pool_dma;
- if (prev == atomic_pool_dma32)
- return atomic_pool_dma;
+
+ if (prev == &atomic_pool_kernel)
+ return __dma_guess_pool(&atomic_pool_dma32,
+ &atomic_pool_dma, NULL);
+
+ if (prev == &atomic_pool_dma32)
+ return __dma_guess_pool(&atomic_pool_dma, NULL, NULL);
+
return NULL;
}
@@ -276,16 +315,20 @@ static struct page *__dma_alloc_from_pool(struct device *dev, size_t size,
}
struct page *dma_alloc_from_pool(struct device *dev, size_t size,
- void **cpu_addr, gfp_t gfp,
+ void **cpu_addr, gfp_t gfp, unsigned long attrs,
bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t))
{
- struct gen_pool *pool = NULL;
+ struct dma_gen_pool *dma_pool = NULL;
struct page *page;
bool pool_found = false;
- while ((pool = dma_guess_pool(pool, gfp))) {
+ while ((dma_pool = dma_guess_pool(dma_pool, gfp))) {
+
+ if (dma_pool->cc_shared != !!(attrs & __DMA_ATTR_ALLOC_CC_SHARED))
+ continue;
+
pool_found = true;
- page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,
+ page = __dma_alloc_from_pool(dev, size, dma_pool->pool, cpu_addr,
phys_addr_ok);
if (page)
return page;
@@ -300,12 +343,14 @@ struct page *dma_alloc_from_pool(struct device *dev, size_t size,
bool dma_free_from_pool(struct device *dev, void *start, size_t size)
{
- struct gen_pool *pool = NULL;
+ struct dma_gen_pool *dma_pool = NULL;
+
+ while ((dma_pool = dma_guess_pool(dma_pool, 0))) {
- while ((pool = dma_guess_pool(pool, 0))) {
- if (!gen_pool_has_addr(pool, (unsigned long)start, size))
+ if (!gen_pool_has_addr(dma_pool->pool, (unsigned long)start, size))
continue;
- gen_pool_free(pool, (unsigned long)start, size);
+
+ gen_pool_free(dma_pool->pool, (unsigned long)start, size);
return true;
}
@@ -337,7 +382,7 @@ static void dma_pool_find_phys(struct gen_pool *pool, struct gen_pool_chunk *chu
match->found = true;
}
-static bool dma_free_from_pool_phys(struct gen_pool *pool, phys_addr_t phys,
+static bool dma_free_from_pool_phys(struct dma_gen_pool *dma_pool, phys_addr_t phys,
size_t size)
{
struct dma_pool_phys_match match = {
@@ -345,21 +390,21 @@ static bool dma_free_from_pool_phys(struct gen_pool *pool, phys_addr_t phys,
.size = size,
};
- gen_pool_for_each_chunk(pool, dma_pool_find_phys, &match);
+ gen_pool_for_each_chunk(dma_pool->pool, dma_pool_find_phys, &match);
if (!match.found)
return false;
- gen_pool_free(pool, match.addr, size);
+ gen_pool_free(dma_pool->pool, match.addr, size);
return true;
}
bool dma_free_from_pool_page(struct device *dev, struct page *page, size_t size)
{
- struct gen_pool *pool = NULL;
+ struct dma_gen_pool *dma_pool = NULL;
phys_addr_t phys = page_to_phys(page);
- while ((pool = dma_guess_pool(pool, 0))) {
- if (dma_free_from_pool_phys(pool, phys, size))
+ while ((dma_pool = dma_guess_pool(dma_pool, 0))) {
+ if (dma_free_from_pool_phys(dma_pool, phys, size))
return true;
}
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index d54154c165e5..908de28aceb2 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -613,9 +613,9 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
u64 phys_limit, gfp_t gfp, void **vaddr)
{
struct page *page;
+ unsigned long attrs = 0;
*vaddr = NULL;
-
/*
* Allocate from the atomic pools if memory is encrypted and
* the allocation is atomic, because decrypting may block.
@@ -624,8 +624,12 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
return NULL;
+ /* swiotlb considered decrypted by default */
+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT))
+ attrs = __DMA_ATTR_ALLOC_CC_SHARED;
+
return dma_alloc_from_pool(dev, bytes, vaddr, gfp,
- dma_coherent_ok);
+ attrs, dma_coherent_ok);
}
gfp &= ~GFP_ZONEMASK;
--
2.43.0
^ permalink raw reply related
* [PATCH v7 10/22] dma-direct: use __DMA_ATTR_ALLOC_CC_SHARED in alloc/free paths
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86, Jiri Pirko,
Michael Kelley
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
Propagate force_dma_unencrypted() into __DMA_ATTR_ALLOC_CC_SHARED in the
dma-direct allocation path and use the attribute to drive the related
decisions.
This updates dma_direct_alloc(), dma_direct_free(), and
dma_direct_alloc_pages() to fold the forced unencrypted case into attrs.
Tested-by: Jiri Pirko <jiri@nvidia.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
kernel/dma/direct.c | 42 +++++++++++++++++++++++++++++++++---------
kernel/dma/mapping.c | 9 +++++++++
2 files changed, 42 insertions(+), 9 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 0cbf2b0835c4..98e47e0b332d 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -192,16 +192,22 @@ void *dma_direct_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
bool remap = false, set_uncached = false;
- bool mark_mem_decrypt = true;
+ bool mark_mem_decrypt = false;
struct page *page;
void *ret;
+ if (force_dma_unencrypted(dev))
+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
+
+ if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
+ mark_mem_decrypt = true;
+
size = PAGE_ALIGN(size);
if (attrs & DMA_ATTR_NO_WARN)
gfp |= __GFP_NOWARN;
- if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
- !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev))
+ if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==
+ DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev))
return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
if (!dev_is_dma_coherent(dev)) {
@@ -235,7 +241,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
* Remapping or decrypting memory may block, allocate the memory from
* the atomic pools instead if we aren't allowed block.
*/
- if ((remap || force_dma_unencrypted(dev)) &&
+ if ((remap || (attrs & __DMA_ATTR_ALLOC_CC_SHARED)) &&
dma_direct_use_pool(dev, gfp)) {
page = dma_direct_alloc_from_pool(dev, size, dma_handle,
&ret, gfp);
@@ -314,12 +320,22 @@ void dma_direct_free(struct device *dev, size_t size,
void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs)
{
phys_addr_t phys;
- bool mark_mem_encrypted = true;
+ bool mark_mem_encrypted = false;
struct io_tlb_pool *swiotlb_pool;
unsigned int page_order = get_order(size);
- if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) &&
- !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) {
+ /*
+ * If the allocation used decrypted/shared backing pages, restore
+ * the encryption state on free.
+ */
+ if (force_dma_unencrypted(dev))
+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
+
+ if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
+ mark_mem_encrypted = true;
+
+ if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==
+ DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) {
/* cpu_addr is a struct page cookie, not a kernel address */
dma_free_contiguous(dev, cpu_addr, size);
return;
@@ -368,10 +384,14 @@ void dma_direct_free(struct device *dev, size_t size,
struct page *dma_direct_alloc_pages(struct device *dev, size_t size,
dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
{
+ unsigned long attrs = 0;
struct page *page;
void *ret;
- if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp))
+ if (force_dma_unencrypted(dev))
+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
+
+ if ((attrs & __DMA_ATTR_ALLOC_CC_SHARED) && dma_direct_use_pool(dev, gfp))
return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp);
if (is_swiotlb_for_alloc(dev)) {
@@ -405,7 +425,11 @@ void dma_direct_free_pages(struct device *dev, size_t size,
phys_addr_t phys;
void *vaddr = page_address(page);
struct io_tlb_pool *swiotlb_pool;
- bool mark_mem_encrypted = true;
+ /*
+ * if the device had requested for an unencrypted buffer,
+ * convert it to encrypted on free
+ */
+ bool mark_mem_encrypted = force_dma_unencrypted(dev);
/* If page is not from an atomic pool, dma_free_from_pool_page() fails */
if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 4fe04669e5e6..d2f70b6ccd0f 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -638,6 +638,15 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
if (WARN_ON_ONCE(flag & __GFP_COMP))
return NULL;
+ if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED)) {
+ trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag,
+ attrs);
+ return NULL;
+ }
+
+ if (force_dma_unencrypted(dev))
+ attrs |= __DMA_ATTR_ALLOC_CC_SHARED;
+
if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {
trace_dma_alloc(dev, cpu_addr, *dma_handle, size,
DMA_BIDIRECTIONAL, flag, attrs);
--
2.43.0
^ permalink raw reply related
* [PATCH v7 09/22] dma-mapping: Add internal shared allocation attribute
From: Aneesh Kumar K.V (Arm) @ 2026-07-01 5:49 UTC (permalink / raw)
To: iommu, linux-arm-kernel, linux-kernel, linux-coco
Cc: Aneesh Kumar K.V (Arm), Robin Murphy, Marek Szyprowski,
Will Deacon, Marc Zyngier, Steven Price, Suzuki K Poulose,
Catalin Marinas, Jiri Pirko, Jason Gunthorpe, Mostafa Saleh,
Petr Tesarik, Alexey Kardashevskiy, Dan Williams, Xu Yilun,
linuxppc-dev, linux-s390, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Alexander Gordeev,
Gerald Schaefer, Heiko Carstens, Vasily Gorbik,
Christian Borntraeger, Sven Schnelle, x86
In-Reply-To: <20260701054926.825925-1-aneesh.kumar@kernel.org>
DMA_ATTR_CC_SHARED describes an existing DMA mapping whose backing memory
is already shared, or decrypted, for confidential computing. It is a
mapping attribute: callers use it to request a shared DMA address encoding
for memory that has already been prepared for shared DMA.
Allocation paths need a related but different state. Once the DMA core
decides that an allocation must use shared backing pages, the lower-level
allocation helpers need to select shared pools, decrypt newly allocated
pages, derive the DMA address with the shared-memory translation and
restore encryption on free. That state is internal to the DMA-mapping
implementation and should not be passed by drivers to dma_alloc_attrs().
Signed-off-by: Aneesh Kumar K.V (Arm) <aneesh.kumar@kernel.org>
---
Documentation/core-api/dma-attributes.rst | 29 +++++++++++++++++++++++
include/linux/dma-mapping.h | 8 +++++++
include/trace/events/dma.h | 3 ++-
3 files changed, 39 insertions(+), 1 deletion(-)
diff --git a/Documentation/core-api/dma-attributes.rst b/Documentation/core-api/dma-attributes.rst
index 123c8468d58f..eee743184acd 100644
--- a/Documentation/core-api/dma-attributes.rst
+++ b/Documentation/core-api/dma-attributes.rst
@@ -179,3 +179,32 @@ interface when building their uAPIs, when possible.
It must never be used in an in-kernel driver that only works with
kernel memory.
+
+DMA_ATTR_CC_SHARED
+------------------
+
+This attribute indicates that a DMA mapping is shared, or decrypted, for
+confidential computing guests. For normal system memory, the caller must
+already have marked the memory decrypted with set_memory_decrypted(). CPU
+PTEs for the mapping must use pgprot_decrypted(), and the same shared
+semantic may be passed to a vIOMMU when it sets up the IOPTE.
+
+This attribute describes an existing mapping. It does not allocate shared
+backing pages and must not be passed to dma_alloc_attrs(). For MMIO, use
+this together with DMA_ATTR_MMIO to indicate shared MMIO. Unless
+DMA_ATTR_MMIO is provided, the mapping requires a struct page.
+
+__DMA_ATTR_ALLOC_CC_SHARED
+--------------------------
+
+This is an internal DMA-mapping attribute for confidential computing guests.
+It is used by allocation paths after the DMA core has determined that the
+backing pages must be shared, or decrypted. For example, the direct DMA and
+SWIOTLB allocation paths use it to select shared DMA pools, decrypt newly
+allocated pages, derive DMA addresses using the shared-memory translation, and
+restore encryption on free.
+
+__DMA_ATTR_ALLOC_CC_SHARED differs from DMA_ATTR_CC_SHARED in that it is not
+a caller-visible DMA API attribute. DMA_ATTR_CC_SHARED describes an
+already-shared mapping and requires the caller to have prepared normal
+system memory before mapping it.
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index cc0823a99cfd..a3e880649fa4 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -103,6 +103,14 @@
*/
#define DMA_ATTR_CC_SHARED (1UL << 13)
+/*
+ * __DMA_ATTR_ALLOC_CC_SHARED: Internal DMA-mapping attribute used by
+ * allocation paths that create shared (decrypted) backing pages for
+ * confidential computing guests. Drivers must not pass this attribute to
+ * dma_alloc_attrs().
+ */
+#define __DMA_ATTR_ALLOC_CC_SHARED (1UL << 14)
+
/*
* A dma_addr_t can hold any valid DMA or bus address for the platform. It can
* be given to a device to use as a DMA source or target. It is specific to a
diff --git a/include/trace/events/dma.h b/include/trace/events/dma.h
index 31c9ddf72c9d..9df02c1511de 100644
--- a/include/trace/events/dma.h
+++ b/include/trace/events/dma.h
@@ -35,7 +35,8 @@ TRACE_DEFINE_ENUM(DMA_NONE);
{ DMA_ATTR_MMIO, "MMIO" }, \
{ DMA_ATTR_DEBUGGING_IGNORE_CACHELINES, "CACHELINES_OVERLAP" }, \
{ DMA_ATTR_REQUIRE_COHERENT, "REQUIRE_COHERENT" }, \
- { DMA_ATTR_CC_SHARED, "CC_SHARED" })
+ { DMA_ATTR_CC_SHARED, "CC_SHARED" }, \
+ { __DMA_ATTR_ALLOC_CC_SHARED, "ALLOC_CC_SHARED" })
DECLARE_EVENT_CLASS(dma_map,
TP_PROTO(struct device *dev, phys_addr_t phys_addr, dma_addr_t dma_addr,
--
2.43.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox