* [PATCH v6 1/8] tracing/probes: Support dumping fetcharg program for debugging dynamic events
From: Masami Hiramatsu (Google) @ 2026-06-20 15:17 UTC (permalink / raw)
To: Steven Rostedt, Mathieu Desnoyers
Cc: Jonathan Corbet, Shuah Khan, Masami Hiramatsu, linux-kernel,
linux-trace-kernel, linux-doc, linux-kselftest
In-Reply-To: <178196862271.560995.5255615288323003663.stgit@devnote2>
From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
For debugging probe events, it is helpful to verify the compiled
fetch instructions for each probe argument. This introduces a new
kernel config CONFIG_PROBE_EVENTS_DUMP_FETCHARG to decode the
instruction sequence of each argument and display it under a
commented line starting with '#' immediately following the dynamic
event definition (such as in dynamic_events, kprobe_events,
uprobe_events, etc.).
For example:
/sys/kernel/tracing # cat dynamic_events
p:kprobes/p_vfs_read_0 vfs_read arg1=+0(file):ustring arg2=%ax:x16
# arg1: ARG(0) -> ST_USTRING(offset=0,size=4) -> END
# arg2: REG(80) -> ST_RAW(size=2) -> END
Assisted-by: Antigravity:gemini-3.5-flash
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
---
Changes in v6:
- Newly added.
---
kernel/trace/Kconfig | 11 +++++
kernel/trace/trace_eprobe.c | 2 +
kernel/trace/trace_fprobe.c | 2 +
kernel/trace/trace_kprobe.c | 2 +
kernel/trace/trace_probe.c | 90 +++++++++++++++++++++++++++++++++++++++++++
kernel/trace/trace_probe.h | 77 ++++++++++++++++++++++---------------
kernel/trace/trace_uprobe.c | 3 +
7 files changed, 157 insertions(+), 30 deletions(-)
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index e130da35808f..ed83fbfb4b7c 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -779,6 +779,17 @@ config PROBE_EVENTS_BTF_ARGS
kernel function entry or a tracepoint.
This is available only if BTF (BPF Type Format) support is enabled.
+config PROBE_EVENTS_DUMP_FETCHARG
+ depends on PROBE_EVENTS
+ bool "Dump of dynamic probe event fetch-arguments"
+ default n
+ help
+ This shows the dump of fetch-arguments of dynamic probe events
+ alongside their event definitions in the dynamic_events file
+ as comment lines. This is useful to debug the probe events.
+
+ If unsure, say N.
+
config KPROBE_EVENTS
depends on KPROBES
depends on HAVE_REGS_AND_STACK_ACCESS_API
diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c
index b66d6196338d..fdb4ce993cad 100644
--- a/kernel/trace/trace_eprobe.c
+++ b/kernel/trace/trace_eprobe.c
@@ -87,6 +87,8 @@ static int eprobe_dyn_event_show(struct seq_file *m, struct dyn_event *ev)
seq_printf(m, " %s=%s", ep->tp.args[i].name, ep->tp.args[i].comm);
seq_putc(m, '\n');
+ trace_probe_dump_args(m, &ep->tp);
+
return 0;
}
diff --git a/kernel/trace/trace_fprobe.c b/kernel/trace/trace_fprobe.c
index 4d1abbf66229..536781cd4c47 100644
--- a/kernel/trace/trace_fprobe.c
+++ b/kernel/trace/trace_fprobe.c
@@ -1449,6 +1449,8 @@ static int trace_fprobe_show(struct seq_file *m, struct dyn_event *ev)
seq_printf(m, " %s=%s", tf->tp.args[i].name, tf->tp.args[i].comm);
seq_putc(m, '\n');
+ trace_probe_dump_args(m, &tf->tp);
+
return 0;
}
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index a8420e6abb56..cfa807d8e760 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -1320,6 +1320,8 @@ static int trace_kprobe_show(struct seq_file *m, struct dyn_event *ev)
seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
seq_putc(m, '\n');
+ trace_probe_dump_args(m, &tk->tp);
+
return 0;
}
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index 98532c503d02..9d174cd1fb1c 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -2393,3 +2393,93 @@ int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_a
}
return 0;
}
+
+#ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG
+
+struct fetch_op_decode {
+ const char *name;
+ void (*decode)(struct seq_file *m, struct fetch_insn *insn);
+};
+
+static const struct fetch_op_decode fetch_op_decode[];
+
+static void fetcharg_decode_none(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_puts(m, fetch_op_decode[insn->op].name);
+}
+
+static void fetcharg_decode_param(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(%u)", fetch_op_decode[insn->op].name, insn->param);
+}
+
+static void fetcharg_decode_imm(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(0x%lx)", fetch_op_decode[insn->op].name, insn->immediate);
+}
+
+static void fetcharg_decode_ptr(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(%p)", fetch_op_decode[insn->op].name, insn->data);
+}
+
+static void fetcharg_decode_symbol(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data);
+}
+
+static void fetcharg_decode_offset(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(offset=%d)", fetch_op_decode[insn->op].name, insn->offset);
+}
+
+static void fetcharg_decode_store(struct seq_file *m, struct fetch_insn *insn)
+{
+ if (insn->op == FETCH_OP_ST_RAW)
+ seq_printf(m, "%s(size=%u)", fetch_op_decode[insn->op].name, insn->size);
+ else
+ seq_printf(m, "%s(offset=%d,size=%u)", fetch_op_decode[insn->op].name, insn->offset, insn->size);
+}
+
+static void fetcharg_decode_bf(struct seq_file *m, struct fetch_insn *insn)
+{
+ seq_printf(m, "%s(basesize=%u,lshift=%u,rshift=%u)",
+ fetch_op_decode[insn->op].name, insn->basesize, insn->lshift, insn->rshift);
+}
+
+#define FETCH_OP(opname, decode_fn) \
+ [FETCH_OP_##opname] = { .name = #opname, .decode = fetcharg_decode_##decode_fn },
+
+static const struct fetch_op_decode fetch_op_decode[] = {
+ FETCH_OP_LIST
+};
+#undef FETCH_OP
+
+static void trace_probe_dump_arg(struct seq_file *m, struct probe_arg *parg)
+{
+ int i;
+
+ seq_printf(m, "# %s: ", parg->name);
+ for (i = 0; i < FETCH_INSN_MAX; i++) {
+ struct fetch_insn *insn = parg->code + i;
+
+ if (insn->op >= ARRAY_SIZE(fetch_op_decode) || !fetch_op_decode[insn->op].decode)
+ seq_printf(m, "unknown(%d)", insn->op);
+ else
+ fetch_op_decode[insn->op].decode(m, insn);
+
+ if (insn->op == FETCH_OP_END)
+ break;
+ seq_puts(m, " -> ");
+ }
+ seq_putc(m, '\n');
+}
+
+void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp)
+{
+ int i;
+
+ for (i = 0; i < tp->nr_args; i++)
+ trace_probe_dump_arg(m, &tp->args[i]);
+}
+#endif /* CONFIG_PROBE_EVENTS_DUMP_FETCHARG */
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index 0f09f7aaf93f..b428ef42b229 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -83,38 +83,47 @@ static nokprobe_inline u32 update_data_loc(u32 loc, int consumed)
/* Printing function type */
typedef int (*print_type_func_t)(struct trace_seq *, void *, void *);
+#define FETCH_OP_LIST \
+ /* Stage 1 (load) ops */ \
+ FETCH_OP(NOP, none) /* NOP */ \
+ FETCH_OP(REG, param) /* Register: .param = offset */ \
+ FETCH_OP(STACK, param) /* Stack: .param = index */ \
+ FETCH_OP(STACKP, none) /* Stack pointer */ \
+ FETCH_OP(RETVAL, none) /* Return value */ \
+ FETCH_OP(IMM, imm) /* Immediate: .immediate */ \
+ FETCH_OP(COMM, none) /* Current comm */ \
+ FETCH_OP(ARG, param) /* Argument: .param = index */ \
+ FETCH_OP(FOFFS, imm) /* File offset: .immediate */ \
+ FETCH_OP(DATA, ptr) /* Allocated data: .data */ \
+ FETCH_OP(EDATA, offset) /* Entry data: .offset */ \
+ FETCH_OP(TP_ARG, param) /* Tracepoint argument: .data */\
+ /* Stage 2 (dereference) ops */ \
+ FETCH_OP(DEREF, offset) /* Dereference: .offset */ \
+ FETCH_OP(UDEREF, offset) /* User-space dereference: .offset */\
+ /* Stage 3 (store) ops */ \
+ FETCH_OP(ST_RAW, store) /* Raw value: .size */ \
+ FETCH_OP(ST_MEM, store) /* Memory: .offset, .size */ \
+ FETCH_OP(ST_UMEM, store) /* User memory: .offset, .size */\
+ FETCH_OP(ST_STRING, store) /* String: .offset, .size */ \
+ FETCH_OP(ST_USTRING, store) /* User string: .offset, .size */\
+ FETCH_OP(ST_SYMSTR, store) /* Symbol name: .offset, .size */\
+ FETCH_OP(ST_EDATA, offset) /* Entry data: .offset */ \
+ /* Stage 4 (modify) op */ \
+ FETCH_OP(MOD_BF, bf) /* Bitfield: .basesize, .lshift, .rshift*/\
+ /* Stage 5 (loop) op */ \
+ FETCH_OP(LP_ARRAY, param) /* Loop array: .param = count */\
+ /* End */ \
+ FETCH_OP(END, none) \
+ /* Unresolved Symbol holder */ \
+ FETCH_OP(NOP_SYMBOL, symbol) /* Non loaded symbol: .data = symbol name */
+
+#define FETCH_OP(opname, decode_fn) FETCH_OP_##opname,
enum fetch_op {
- FETCH_OP_NOP = 0,
- // Stage 1 (load) ops
- FETCH_OP_REG, /* Register : .param = offset */
- FETCH_OP_STACK, /* Stack : .param = index */
- FETCH_OP_STACKP, /* Stack pointer */
- FETCH_OP_RETVAL, /* Return value */
- FETCH_OP_IMM, /* Immediate : .immediate */
- FETCH_OP_COMM, /* Current comm */
- FETCH_OP_ARG, /* Function argument : .param */
- FETCH_OP_FOFFS, /* File offset: .immediate */
- FETCH_OP_DATA, /* Allocated data: .data */
- FETCH_OP_EDATA, /* Entry data: .offset */
- // Stage 2 (dereference) op
- FETCH_OP_DEREF, /* Dereference: .offset */
- FETCH_OP_UDEREF, /* User-space Dereference: .offset */
- // Stage 3 (store) ops
- FETCH_OP_ST_RAW, /* Raw: .size */
- FETCH_OP_ST_MEM, /* Mem: .offset, .size */
- FETCH_OP_ST_UMEM, /* Mem: .offset, .size */
- FETCH_OP_ST_STRING, /* String: .offset, .size */
- FETCH_OP_ST_USTRING, /* User String: .offset, .size */
- FETCH_OP_ST_SYMSTR, /* Kernel Symbol String: .offset, .size */
- FETCH_OP_ST_EDATA, /* Store Entry Data: .offset */
- // Stage 4 (modify) op
- FETCH_OP_MOD_BF, /* Bitfield: .basesize, .lshift, .rshift */
- // Stage 5 (loop) op
- FETCH_OP_LP_ARRAY, /* Array: .param = loop count */
- FETCH_OP_TP_ARG, /* Trace Point argument */
- FETCH_OP_END,
- FETCH_NOP_SYMBOL, /* Unresolved Symbol holder */
+ FETCH_OP_LIST
};
+#undef FETCH_OP
+
+#define FETCH_NOP_SYMBOL FETCH_OP_NOP_SYMBOL
struct fetch_insn {
enum fetch_op op;
@@ -370,6 +379,14 @@ bool trace_probe_match_command_args(struct trace_probe *tp,
int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **));
int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
u8 *data, void *field);
+#ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG
+void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp);
+#else
+static inline void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp)
+{
+ return;
+}
+#endif
#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
int traceprobe_get_entry_data_size(struct trace_probe *tp);
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index c274346853d1..b2e264a4b96c 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -765,6 +765,9 @@ static int trace_uprobe_show(struct seq_file *m, struct dyn_event *ev)
seq_printf(m, " %s=%s", tu->tp.args[i].name, tu->tp.args[i].comm);
seq_putc(m, '\n');
+
+ trace_probe_dump_args(m, &tu->tp);
+
return 0;
}
^ permalink raw reply related
* [PATCH v6 2/8] tracing/probes: Support typecast for various probe events
From: Masami Hiramatsu (Google) @ 2026-06-20 15:17 UTC (permalink / raw)
To: Steven Rostedt, Mathieu Desnoyers
Cc: Jonathan Corbet, Shuah Khan, Masami Hiramatsu, linux-kernel,
linux-trace-kernel, linux-doc, linux-kselftest
In-Reply-To: <178196862271.560995.5255615288323003663.stgit@devnote2>
From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Support BTF typecast feature on other probe events, but only if it is
kernel function entry or return, and must use function parameter name
or $retval. This means you can do:
(STRUCT)PARAM->MEMBER
Note: you can not use other variables like $stackN, %reg etc. That
needs nesting support.
To support other probe events, we just need to use last_struct type
when we find a function parameter in parse_btf_arg().
This also updates <tracefs>/README file to show struct typecast.
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
---
Changes in v5:
- Add comments about $retval with typecast.
- Even if the type of retvalue is not known, if user specifies typecast,
use it for its type.
Changes in v3:
- Clarify the limitation.
Changes in v2:
- Fix to re-enable typecast on eprobe.
---
Documentation/trace/fprobetrace.rst | 3 +++
Documentation/trace/kprobetrace.rst | 4 ++++
kernel/trace/trace.c | 2 +-
kernel/trace/trace_probe.c | 23 +++++++++++++++++------
kernel/trace/trace_probe.h | 5 +++++
5 files changed, 30 insertions(+), 7 deletions(-)
diff --git a/Documentation/trace/fprobetrace.rst b/Documentation/trace/fprobetrace.rst
index b4c2ca3d02c1..7435ded2d66d 100644
--- a/Documentation/trace/fprobetrace.rst
+++ b/Documentation/trace/fprobetrace.rst
@@ -57,6 +57,9 @@ Synopsis of fprobe-events
(u8/u16/u32/u64/s8/s16/s32/s64), hexadecimal types
(x8/x16/x32/x64), "char", "string", "ustring", "symbol", "symstr"
and bitfield are supported.
+ (STRUCT)FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
+ a pointer to STRUCT and then derference the pointer defined by
+ ->MEMBER.
(\*1) This is available only when BTF is enabled.
(\*2) only for the probe on function entry (offs == 0). Note, this argument access
diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst
index 3b6791c17e9b..f73614997d52 100644
--- a/Documentation/trace/kprobetrace.rst
+++ b/Documentation/trace/kprobetrace.rst
@@ -61,6 +61,10 @@ Synopsis of kprobe_events
(x8/x16/x32/x64), VFS layer common type(%pd/%pD), "char",
"string", "ustring", "symbol", "symstr" and bitfield are
supported.
+ (STRUCT)FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
+ a pointer to STRUCT and then derference the pointer defined by
+ ->MEMBER. Note that this is available only when the probe is
+ on function entry.
(\*1) only for the probe on function entry (offs == 0). Note, this argument access
is best effort, because depending on the argument type, it may be passed on
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 6eb4d3097a4d..aa93e7b01146 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -4325,7 +4325,7 @@ static const char readme_msg[] =
#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
"\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
- "\t <argname>[->field[->field|.field...]],\n"
+ "\t [(structname)]<argname>[->field[->field|.field...]],\n"
#endif
#else
"\t $stack<index>, $stack, $retval, $comm,\n"
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index 9d174cd1fb1c..76ee3ca48d6a 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -706,7 +706,7 @@ static int parse_btf_arg(char *varname,
if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
code->op = FETCH_OP_RETVAL;
- /* Check whether the function return type is not void */
+ /* Check whether the function return type is not void, even with typecast. */
if (query_btf_context(ctx) == 0) {
if (ctx->proto->type == 0) {
trace_probe_log_err(ctx->offset, NO_RETVAL);
@@ -715,6 +715,13 @@ static int parse_btf_arg(char *varname,
tid = ctx->proto->type;
goto found;
}
+ /*
+ * Even if we can not find appropriate BTF info, we can still access
+ * the field via typecast.
+ */
+ if (ctx->struct_btf)
+ goto found;
+
if (field) {
trace_probe_log_err(ctx->offset + field - varname,
NO_BTF_ENTRY);
@@ -759,7 +766,10 @@ static int parse_btf_arg(char *varname,
return -ENOENT;
found:
- type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
+ if (ctx->struct_btf)
+ type = ctx->last_struct;
+ else
+ type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
found_type:
if (!type) {
trace_probe_log_err(ctx->offset, BAD_BTF_TID);
@@ -836,10 +846,11 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode,
char *tmp;
int ret;
- /* Currently this only works for eprobes */
- if (!(ctx->flags & TPARG_FL_TEVENT)) {
- trace_probe_log_err(ctx->offset, TYPECAST_NOT_EVENT);
- return -EINVAL;
+ if (!(tparg_is_event_probe(ctx->flags) ||
+ tparg_is_function_entry(ctx->flags) ||
+ tparg_is_function_return(ctx->flags))) {
+ trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
+ return -EOPNOTSUPP;
}
tmp = strchr(arg, ')');
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index b428ef42b229..e112424f3529 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -431,6 +431,11 @@ static inline bool tparg_is_function_return(unsigned int flags)
return (flags & TPARG_FL_LOC_MASK) == (TPARG_FL_KERNEL | TPARG_FL_RETURN);
}
+static inline bool tparg_is_event_probe(unsigned int flags)
+{
+ return !!(flags & TPARG_FL_TEVENT);
+}
+
struct traceprobe_parse_context {
struct trace_event_call *event;
/* BTF related parameters */
^ permalink raw reply related
* [PATCH v6 3/8] tracing/probes: Support nested typecast
From: Masami Hiramatsu (Google) @ 2026-06-20 15:17 UTC (permalink / raw)
To: Steven Rostedt, Mathieu Desnoyers
Cc: Jonathan Corbet, Shuah Khan, Masami Hiramatsu, linux-kernel,
linux-trace-kernel, linux-doc, linux-kselftest
In-Reply-To: <178196862271.560995.5255615288323003663.stgit@devnote2>
From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
When we hit an open parenthesis right after typecast closing
parenthesis, it means we have nested typecast. This allows us to
typecast a generic data member in a structure to a pointer to
another structure.
For example, to cast a DATA_MEMBER of VAR structure to STRUCT pointer
and get MEMBER value.
(STRUCT)(VAR->DATA_MEMBER)->MEMBER
Also, we can nest typecast.
(STRUCT1)((STRUCT2)$ARG->FIELD2)->FIELD1
Currently the max nest level is limited to 3.
This also allows user to use typecasting for registers or stacks on
kprobe events. e.g.
(STRUCT)(%ax)->MEMBER
(STRUCT)($stack0)->MEMBER
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
---
Changes in v6:
- Add a WARN_ON_ONCE check for leaking nested_level (it must not happen.)
Changes in v4:
- Use orig_offset for reporting NO_PTR_STRCT error.
Changes in v2:
- Fix to skip "->" after closing parenthetsis.
---
Documentation/trace/eprobetrace.rst | 2 +
Documentation/trace/fprobetrace.rst | 2 +
Documentation/trace/kprobetrace.rst | 2 +
kernel/trace/trace.c | 1
kernel/trace/trace_probe.c | 81 ++++++++++++++++++++++++++++++++---
kernel/trace/trace_probe.h | 7 +++
6 files changed, 86 insertions(+), 9 deletions(-)
diff --git a/Documentation/trace/eprobetrace.rst b/Documentation/trace/eprobetrace.rst
index fe3602540569..cd0b4aa7f896 100644
--- a/Documentation/trace/eprobetrace.rst
+++ b/Documentation/trace/eprobetrace.rst
@@ -50,6 +50,8 @@ Synopsis of eprobe_events
a pointer to STRUCT and then derference the pointer defined by
->MEMBER. Note that when this is used, the FIELD name does not
need to be prefixed with a '$'.
+ (STRUCT)(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
+ also be used with another FETCHARG instead of FIELD.
Types
-----
diff --git a/Documentation/trace/fprobetrace.rst b/Documentation/trace/fprobetrace.rst
index 7435ded2d66d..6b8bb27bb62d 100644
--- a/Documentation/trace/fprobetrace.rst
+++ b/Documentation/trace/fprobetrace.rst
@@ -60,6 +60,8 @@ Synopsis of fprobe-events
(STRUCT)FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
a pointer to STRUCT and then derference the pointer defined by
->MEMBER.
+ (STRUCT)(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
+ also be used with another FETCHARG instead of FIELD.
(\*1) This is available only when BTF is enabled.
(\*2) only for the probe on function entry (offs == 0). Note, this argument access
diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst
index f73614997d52..c4382765d5b2 100644
--- a/Documentation/trace/kprobetrace.rst
+++ b/Documentation/trace/kprobetrace.rst
@@ -65,6 +65,8 @@ Synopsis of kprobe_events
a pointer to STRUCT and then derference the pointer defined by
->MEMBER. Note that this is available only when the probe is
on function entry.
+ (STRUCT)(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
+ also be used with another FETCHARG instead of FIELD.
(\*1) only for the probe on function entry (offs == 0). Note, this argument access
is best effort, because depending on the argument type, it may be passed on
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index aa93e7b01146..4f70318918c2 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -4326,6 +4326,7 @@ static const char readme_msg[] =
"\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
"\t [(structname)]<argname>[->field[->field|.field...]],\n"
+ "\t [(structname)](fetcharg)->field[->field|.field...],\n"
#endif
#else
"\t $stack<index>, $stack, $retval, $comm,\n"
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index 76ee3ca48d6a..cebfba580922 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -839,10 +839,35 @@ static int query_btf_struct(const char *sname, struct traceprobe_parse_context *
return 0;
}
+/* Find the matching closing parenthesis for a given opening parenthesis. */
+static char *find_matched_close_paren(char *s)
+{
+ char *p = s;
+ int count = 0;
+
+ while (*p) {
+ if (*p == '(')
+ count++;
+ else if (*p == ')') {
+ if (--count == 0)
+ return p;
+ }
+ p++;
+ }
+ return NULL;
+}
+
+static int
+parse_probe_arg(char *arg, const struct fetch_type *type,
+ struct fetch_insn **pcode, struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx);
+
static int handle_typecast(char *arg, struct fetch_insn **pcode,
struct fetch_insn *end,
struct traceprobe_parse_context *ctx)
{
+ int orig_offset = ctx->offset;
+ bool nested = false;
char *tmp;
int ret;
@@ -859,19 +884,56 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode,
DEREF_OPEN_BRACE);
return -EINVAL;
}
- *tmp = '\0';
- ret = query_btf_struct(arg + 1, ctx);
- *tmp = ')';
+ *tmp++ = '\0';
+
+ /* Handle the nested structure like (STRUCT)(VAR->FIELD)->... */
+ if (*tmp == '(') {
+ char *close = find_matched_close_paren(tmp);
+ ctx->offset += tmp - arg;
+ if (!close) {
+ trace_probe_log_err(ctx->offset, DEREF_OPEN_BRACE);
+ return -EINVAL;
+ }
+ /* We expect a field access for typecast */
+ if (close[1] != '-' || close[2] != '>') {
+ trace_probe_log_err(ctx->offset + close - tmp + 1,
+ TYPECAST_REQ_FIELD);
+ return -EINVAL;
+ }
+
+ ctx->nested_level++;
+ if (ctx->nested_level > TRACEPROBE_MAX_NESTED_LEVEL) {
+ trace_probe_log_err(ctx->offset, TOO_MANY_NESTED);
+ return -E2BIG;
+ }
+ *close = '\0';
+
+ ctx->offset += 1; /* for the '(' */
+ /* We need to parse the nested one */
+ ret = parse_probe_arg(tmp + 1, find_fetch_type(NULL, ctx->flags),
+ pcode, end, ctx);
+ if (ret < 0)
+ return ret;
+ ctx->nested_level--;
+ clear_struct_btf(ctx);
+
+ tmp = close + 3;/* Skip "->" after closing parenthesis */
+ nested = true;
+ }
+
+ ret = query_btf_struct(arg + 1, ctx);
if (ret < 0) {
- trace_probe_log_err(ctx->offset + 1, NO_PTR_STRCT);
+ trace_probe_log_err(orig_offset + 1, NO_PTR_STRCT);
return -EINVAL;
}
- tmp++;
-
- ctx->offset += tmp - arg;
- ret = parse_btf_arg(tmp, pcode, end, ctx);
+ ctx->offset = orig_offset + tmp - arg;
+ /* If it is nested, tmp points to the field name. */
+ if (nested)
+ ret = parse_btf_field(tmp, ctx->last_struct, pcode, end, ctx);
+ else
+ ret = parse_btf_arg(tmp, pcode, end, ctx);
return ret;
}
@@ -1628,6 +1690,9 @@ static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
ctx);
if (ret < 0)
goto fail;
+ /* nested_level must be 0 here, otherwise there is a bug. */
+ if (WARN_ON_ONCE(ctx->nested_level))
+ goto fail;
/* Update storing type if BTF is available */
if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index e112424f3529..1515b3dda5be 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -452,8 +452,11 @@ struct traceprobe_parse_context {
struct trace_probe *tp;
unsigned int flags;
int offset;
+ int nested_level;
};
+#define TRACEPROBE_MAX_NESTED_LEVEL 3
+
extern int traceprobe_parse_probe_arg(struct trace_probe *tp, int i,
const char *argv,
struct traceprobe_parse_context *ctx);
@@ -588,7 +591,9 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
C(TOO_MANY_ARGS, "Too many arguments are specified"), \
C(TOO_MANY_EARGS, "Too many entry arguments specified"), \
C(EVENT_TOO_BIG, "Event too big (too many fields?)"), \
- C(TYPECAST_NOT_EVENT, "Typecasts are only for eprobe fields"),
+ C(TYPECAST_NOT_EVENT, "Typecasts are only for eprobe fields"), \
+ C(TYPECAST_REQ_FIELD, "Typecast requires a field access"), \
+ C(TOO_MANY_NESTED, "Too many nested typecasts/dereferences"),
#undef C
#define C(a, b) TP_ERR_##a
^ permalink raw reply related
* [PATCH v6 4/8] tracing/probes: Type casting always involves nested calls
From: Masami Hiramatsu (Google) @ 2026-06-20 15:17 UTC (permalink / raw)
To: Steven Rostedt, Mathieu Desnoyers
Cc: Jonathan Corbet, Shuah Khan, Masami Hiramatsu, linux-kernel,
linux-trace-kernel, linux-doc, linux-kselftest
In-Reply-To: <178196862271.560995.5255615288323003663.stgit@devnote2>
From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
This allows type casting to various fetchargs without parentheses
by recursively calling parse_probe_arg on the target when type
casting is used.
For example, this allows the following expressions:
- (STRUCT)%REG->FIELD
- (STRUCT)$stackN->FIELD
- (STRUCT)@SYM->FIELD
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
---
Changes in v6:
- Newly added.
---
kernel/trace/trace_probe.c | 101 +++++++++++++++++++++++++++-----------------
kernel/trace/trace_probe.h | 1
2 files changed, 63 insertions(+), 39 deletions(-)
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index cebfba580922..b413bbe8c3af 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -691,19 +691,6 @@ static int parse_btf_arg(char *varname,
return -EOPNOTSUPP;
}
- if (ctx->flags & TPARG_FL_TEVENT) {
- ret = parse_trace_event(varname, code, ctx);
- if (ret < 0) {
- trace_probe_log_err(ctx->offset, BAD_ATTACH_ARG);
- return ret;
- }
- /* TEVENT is only here via a typecast */
- if (WARN_ON_ONCE(ctx->struct_btf == NULL))
- return -EINVAL;
- type = ctx->last_struct;
- goto found_type;
- }
-
if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
code->op = FETCH_OP_RETVAL;
/* Check whether the function return type is not void, even with typecast. */
@@ -867,7 +854,7 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode,
struct traceprobe_parse_context *ctx)
{
int orig_offset = ctx->offset;
- bool nested = false;
+ char *close;
char *tmp;
int ret;
@@ -878,6 +865,17 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode,
return -EOPNOTSUPP;
}
+ /*
+ * Always consider the token after typecast as a nested call
+ * For example: (STRUCT)VAR->FIELD and (STRUCT)(VAR)->FIELD are same.
+ * VAR is solved in the nested call.
+ */
+ ctx->nested_level++;
+ if (ctx->nested_level > TRACEPROBE_MAX_NESTED_LEVEL) {
+ trace_probe_log_err(ctx->offset, TOO_MANY_NESTED);
+ return -E2BIG;
+ }
+
tmp = strchr(arg, ')');
if (!tmp) {
trace_probe_log_err(ctx->offset + strlen(arg),
@@ -886,11 +884,10 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode,
}
*tmp++ = '\0';
- /* Handle the nested structure like (STRUCT)(VAR->FIELD)->... */
+ ctx->offset += tmp - arg;
if (*tmp == '(') {
- char *close = find_matched_close_paren(tmp);
+ close = find_matched_close_paren(tmp);
- ctx->offset += tmp - arg;
if (!close) {
trace_probe_log_err(ctx->offset, DEREF_OPEN_BRACE);
return -EINVAL;
@@ -901,27 +898,57 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode,
TYPECAST_REQ_FIELD);
return -EINVAL;
}
-
- ctx->nested_level++;
- if (ctx->nested_level > TRACEPROBE_MAX_NESTED_LEVEL) {
- trace_probe_log_err(ctx->offset, TOO_MANY_NESTED);
- return -E2BIG;
+ /* Skip '(' */
+ ctx->offset += 1;
+ tmp++;
+ } else if (*tmp == '+' || *tmp == '-') {
+ /* Dereference can have another field access inside it. */
+ char *open = strchr(tmp + 1, '(');
+
+ if (!open) {
+ trace_probe_log_err(ctx->offset,
+ DEREF_NEED_BRACE);
+ return -EINVAL;
}
- *close = '\0';
+ close = find_matched_close_paren(open);
+ if (!close) {
+ trace_probe_log_err(ctx->offset + strlen(tmp),
+ DEREF_OPEN_BRACE);
+ return -EINVAL;
+ }
+ close++;
+ /* We expect a field access for typecast */
+ if (close[0] != '-' || close[1] != '>') {
+ trace_probe_log_err(ctx->offset + close - tmp + 1,
+ TYPECAST_REQ_FIELD);
+ return -EINVAL;
+ }
+ } else {
+ /* Inner variable name */
+ close = strchr(tmp, '-');
+ if (!close || close[1] != '>') {
+ trace_probe_log_err(ctx->offset + strlen(tmp),
+ TYPECAST_REQ_FIELD);
+ return -EINVAL;
+ }
+ }
+ *close = '\0';
- ctx->offset += 1; /* for the '(' */
- /* We need to parse the nested one */
- ret = parse_probe_arg(tmp + 1, find_fetch_type(NULL, ctx->flags),
- pcode, end, ctx);
- if (ret < 0)
- return ret;
- ctx->nested_level--;
- clear_struct_btf(ctx);
+ /* We need to parse the nested one */
+ ret = parse_probe_arg(tmp, find_fetch_type(NULL, ctx->flags),
+ pcode, end, ctx);
+ if (ret < 0)
+ return ret;
+ ctx->nested_level--;
+ clear_struct_btf(ctx);
- tmp = close + 3;/* Skip "->" after closing parenthesis */
- nested = true;
- }
+ /* Let tmp point the field name. */
+ if (close[1] == '-')
+ tmp = close + 3; /* Skip "->" after closing parenthesis */
+ else
+ tmp = close + 2; /* Skip ">" after inner variable name */
+ /* resolve the typecast struct name */
ret = query_btf_struct(arg + 1, ctx);
if (ret < 0) {
trace_probe_log_err(orig_offset + 1, NO_PTR_STRCT);
@@ -929,11 +956,7 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode,
}
ctx->offset = orig_offset + tmp - arg;
- /* If it is nested, tmp points to the field name. */
- if (nested)
- ret = parse_btf_field(tmp, ctx->last_struct, pcode, end, ctx);
- else
- ret = parse_btf_arg(tmp, pcode, end, ctx);
+ ret = parse_btf_field(tmp, ctx->last_struct, pcode, end, ctx);
return ret;
}
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index 1515b3dda5be..e66e0fcb91a3 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -455,6 +455,7 @@ struct traceprobe_parse_context {
int nested_level;
};
+/* Each typecast consumes nested level. So the max number of typecast is 3. */
#define TRACEPROBE_MAX_NESTED_LEVEL 3
extern int traceprobe_parse_probe_arg(struct trace_probe *tp, int i,
^ permalink raw reply related
* [PATCH v6 5/8] tracing/probes: Support field specifier option for typecast
From: Masami Hiramatsu (Google) @ 2026-06-20 15:17 UTC (permalink / raw)
To: Steven Rostedt, Mathieu Desnoyers
Cc: Jonathan Corbet, Shuah Khan, Masami Hiramatsu, linux-kernel,
linux-trace-kernel, linux-doc, linux-kselftest
In-Reply-To: <178196862271.560995.5255615288323003663.stgit@devnote2>
From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Add a field specifier option for the typecast. This works like
container_of() macro.
(STRUCT[,FIELD[.FIELD2...]])VAR
This is equivalent to :
container_of(VAR, struct STRUCT, FIELD[.FIELD2...])
For example:
echo "f tick_nohz_handler next_tick=(tick_sched,sched_timer)timer->next_tick" >> dynamic_events
This will trace tick_nohz_handler() with its tick_sched::next_tick which
is converted from @timer by contianer_of(tick, struct tick_sched, sched_timer).
So, if you enabkle both fprobes:tick_nohz_handler__entry and
timer:hrtimer_expire_entry events, we will see something like:
<idle>-0 [002] d.h1. 3778.087272: hrtimer_expire_entry: hrtimer=00000000d63db328 f
unction=tick_nohz_handler now=3777450051040
<idle>-0 [002] d.h1. 3778.087281: tick_nohz_handler__entry: (tick_nohz_handler+0x4
/0x140) next_tick=3777450000000
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
---
Changes in v6:
- Update according to the allways nested patch.
Changes in v3:
- Fix error caret position.
Changes in v2:
- Use byteoffset for typecast field offset instead of bitoffset. This fixes negative modulo calculation.
- Check whether a field is specified after typecast.
- Reject if typecast field option has arrow operator.
---
Documentation/trace/eprobetrace.rst | 5 +
Documentation/trace/fprobetrace.rst | 8 +-
Documentation/trace/kprobetrace.rst | 8 +-
kernel/trace/trace.c | 4 -
kernel/trace/trace_probe.c | 171 ++++++++++++++++++++++++-----------
kernel/trace/trace_probe.h | 5 +
6 files changed, 136 insertions(+), 65 deletions(-)
diff --git a/Documentation/trace/eprobetrace.rst b/Documentation/trace/eprobetrace.rst
index cd0b4aa7f896..680e0af43d5d 100644
--- a/Documentation/trace/eprobetrace.rst
+++ b/Documentation/trace/eprobetrace.rst
@@ -49,7 +49,10 @@ Synopsis of eprobe_events
(STRUCT)FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
a pointer to STRUCT and then derference the pointer defined by
->MEMBER. Note that when this is used, the FIELD name does not
- need to be prefixed with a '$'.
+ need to be prefixed with a '$'. ASGN can be specified optionally.
+ If ASGN is specified, FIELD will be cast to the same offset
+ position as the ASGN member, rather than to the beginning of
+ the STRUCT.
(STRUCT)(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
also be used with another FETCHARG instead of FIELD.
diff --git a/Documentation/trace/fprobetrace.rst b/Documentation/trace/fprobetrace.rst
index 6b8bb27bb62d..290a9e6f7491 100644
--- a/Documentation/trace/fprobetrace.rst
+++ b/Documentation/trace/fprobetrace.rst
@@ -57,10 +57,12 @@ Synopsis of fprobe-events
(u8/u16/u32/u64/s8/s16/s32/s64), hexadecimal types
(x8/x16/x32/x64), "char", "string", "ustring", "symbol", "symstr"
and bitfield are supported.
- (STRUCT)FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
+ (STRUCT[,ASGN])FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
a pointer to STRUCT and then derference the pointer defined by
- ->MEMBER.
- (STRUCT)(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
+ ->MEMBER. ASGN can be specified optionally. If ASGN is specified,
+ FIELD will be cast to the same offset position as the ASGN member,
+ rather than to the beginning of the STRUCT.
+ (STRUCT[,ASGN])(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
also be used with another FETCHARG instead of FIELD.
(\*1) This is available only when BTF is enabled.
diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst
index c4382765d5b2..a62707e6a9f2 100644
--- a/Documentation/trace/kprobetrace.rst
+++ b/Documentation/trace/kprobetrace.rst
@@ -61,11 +61,13 @@ Synopsis of kprobe_events
(x8/x16/x32/x64), VFS layer common type(%pd/%pD), "char",
"string", "ustring", "symbol", "symstr" and bitfield are
supported.
- (STRUCT)FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
+ (STRUCT[,ASGN])FIELD->MEMBER[->MEMBER] : If BTF is supported, typecast FIELD to
a pointer to STRUCT and then derference the pointer defined by
->MEMBER. Note that this is available only when the probe is
- on function entry.
- (STRUCT)(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
+ on function entry. ASGN can be specified optionally. If ASGN
+ is specified, FIELD will be cast to the same offset position
+ as the ASGN member, rather than to the beginning of the STRUCT.
+ (STRUCT[,ASGN])(FETCHARG)->MEMBER[->MEMBER] : typecast can nest, so the above can
also be used with another FETCHARG instead of FIELD.
(\*1) only for the probe on function entry (offs == 0). Note, this argument access
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 4f70318918c2..0e36af853199 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -4325,8 +4325,8 @@ static const char readme_msg[] =
#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
"\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
- "\t [(structname)]<argname>[->field[->field|.field...]],\n"
- "\t [(structname)](fetcharg)->field[->field|.field...],\n"
+ "\t [(structname[,field])]<argname>[->field[->field|.field...]],\n"
+ "\t [(structname[,field])](fetcharg)->field[->field|.field...],\n"
#endif
#else
"\t $stack<index>, $stack, $retval, $comm,\n"
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index b413bbe8c3af..8c21c378fac7 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -574,6 +574,65 @@ static int split_next_field(char *varname, char **next_field,
return ret;
}
+/* Inner loop for solving dot operator ('.'). Return bit-offset of the given field */
+static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **ptype,
+ struct traceprobe_parse_context *ctx)
+{
+ const struct btf_type *type = *ptype;
+ const struct btf_member *field;
+ struct btf *btf = ctx_btf(ctx);
+ char *fieldname = *pfieldname;
+ int bitoffs = 0;
+ u32 anon_offs;
+ char *next;
+ int is_ptr;
+ s32 tid;
+
+ do {
+ next = NULL;
+ is_ptr = split_next_field(fieldname, &next, ctx);
+ if (is_ptr < 0)
+ return is_ptr;
+
+ anon_offs = 0;
+ field = btf_find_struct_member(btf, type, fieldname,
+ &anon_offs);
+ if (IS_ERR(field)) {
+ trace_probe_log_err(ctx->offset, BAD_BTF_TID);
+ return PTR_ERR(field);
+ }
+ if (!field) {
+ trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
+ return -ENOENT;
+ }
+ /* Add anonymous structure/union offset */
+ bitoffs += anon_offs;
+
+ /* Accumulate the bit-offsets of the dot-connected fields */
+ if (btf_type_kflag(type)) {
+ bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
+ ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
+ } else {
+ bitoffs += field->offset;
+ ctx->last_bitsize = 0;
+ }
+
+ type = btf_type_skip_modifiers(btf, field->type, &tid);
+ if (!type) {
+ trace_probe_log_err(ctx->offset, BAD_BTF_TID);
+ return -EINVAL;
+ }
+
+ if (next)
+ ctx->offset += next - fieldname;
+ fieldname = next;
+ } while (!is_ptr && fieldname);
+
+ *pfieldname = fieldname;
+ *ptype = type;
+
+ return bitoffs;
+}
/*
* Parse the field of data structure. The @type must be a pointer type
* pointing the target data structure type.
@@ -583,16 +642,14 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type,
struct traceprobe_parse_context *ctx)
{
struct fetch_insn *code = *pcode;
- const struct btf_member *field;
- u32 bitoffs, anon_offs;
- bool is_struct = ctx->struct_btf != NULL;
struct btf *btf = ctx_btf(ctx);
- char *next;
- int is_ptr;
+ bool is_first_field = true;
+ int bitoffs;
s32 tid;
do {
- if (!is_struct) {
+ /* For the first field of typecast, @type will be the target structure type. */
+ if (!(is_first_field && ctx->struct_btf)) {
/* Outer loop for solving arrow operator ('->') */
if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
@@ -606,60 +663,25 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type,
return -EINVAL;
}
}
- /* Only the first type can skip being a pointer */
- is_struct = false;
-
- bitoffs = 0;
- do {
- /* Inner loop for solving dot operator ('.') */
- next = NULL;
- is_ptr = split_next_field(fieldname, &next, ctx);
- if (is_ptr < 0)
- return is_ptr;
-
- anon_offs = 0;
- field = btf_find_struct_member(btf, type, fieldname,
- &anon_offs);
- if (IS_ERR(field)) {
- trace_probe_log_err(ctx->offset, BAD_BTF_TID);
- return PTR_ERR(field);
- }
- if (!field) {
- trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
- return -ENOENT;
- }
- /* Add anonymous structure/union offset */
- bitoffs += anon_offs;
-
- /* Accumulate the bit-offsets of the dot-connected fields */
- if (btf_type_kflag(type)) {
- bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
- ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
- } else {
- bitoffs += field->offset;
- ctx->last_bitsize = 0;
- }
-
- type = btf_type_skip_modifiers(btf, field->type, &tid);
- if (!type) {
- trace_probe_log_err(ctx->offset, BAD_BTF_TID);
- return -EINVAL;
- }
-
- ctx->offset += next - fieldname;
- fieldname = next;
- } while (!is_ptr && fieldname);
+ bitoffs = get_bitoffset_of_field(&fieldname, &type, ctx);
+ if (bitoffs < 0)
+ return bitoffs;
if (++code == end) {
trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
return -EINVAL;
}
code->op = FETCH_OP_DEREF; /* TODO: user deref support */
code->offset = bitoffs / 8;
+ if (is_first_field && ctx->struct_btf) {
+ /* The first field can be typecasted with field option. */
+ code->offset -= ctx->prefix_byteoffs;
+ }
*pcode = code;
ctx->last_bitoffs = bitoffs % 8;
ctx->last_type = type;
+ is_first_field = false;
} while (fieldname);
return 0;
@@ -757,7 +779,6 @@ static int parse_btf_arg(char *varname,
type = ctx->last_struct;
else
type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
-found_type:
if (!type) {
trace_probe_log_err(ctx->offset, BAD_BTF_TID);
return -EINVAL;
@@ -826,6 +847,46 @@ static int query_btf_struct(const char *sname, struct traceprobe_parse_context *
return 0;
}
+static int parse_btf_casttype(char *casttype, struct traceprobe_parse_context *ctx)
+{
+ char *field;
+ int ret;
+
+ /* Field option - evaluated later. */
+ field = strchr(casttype, ',');
+ if (field)
+ *field++ = '\0';
+
+ ret = query_btf_struct(casttype, ctx);
+ if (ret < 0) {
+ trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
+ return -EINVAL;
+ }
+
+ if (field) {
+ struct btf_type *type = (struct btf_type *)ctx->last_struct;
+
+ ctx->offset += field - casttype;
+ ret = get_bitoffset_of_field(&field, &ctx->last_struct, ctx);
+ if (ret < 0)
+ return ret;
+ if (ret % 8) {
+ trace_probe_log_err(ctx->offset, TYPECAST_NOT_ALIGNED);
+ return -EINVAL;
+ }
+ if (field != NULL) {
+ /* this means @field skips an arrow operator ("->"). */
+ trace_probe_log_err(ctx->offset - 2, TYPECAST_BAD_ARROW);
+ return -EINVAL;
+ }
+ ctx->prefix_byteoffs = ret / 8;
+ /* Restore the original struct type (overwritten by get_bitoffset_of_field) */
+ ctx->last_struct = type;
+ }
+
+ return ret;
+}
+
/* Find the matching closing parenthesis for a given opening parenthesis. */
static char *find_matched_close_paren(char *s)
{
@@ -949,14 +1010,14 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode,
tmp = close + 2; /* Skip ">" after inner variable name */
/* resolve the typecast struct name */
- ret = query_btf_struct(arg + 1, ctx);
- if (ret < 0) {
- trace_probe_log_err(orig_offset + 1, NO_PTR_STRCT);
- return -EINVAL;
- }
+ ctx->offset = orig_offset + 1; /* for the '(' */
+ ret = parse_btf_casttype(arg + 1, ctx);
+ if (ret < 0)
+ return ret;
ctx->offset = orig_offset + tmp - arg;
ret = parse_btf_field(tmp, ctx->last_struct, pcode, end, ctx);
+ ctx->prefix_byteoffs = 0;
return ret;
}
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index e66e0fcb91a3..bc3ac148a655 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -453,6 +453,7 @@ struct traceprobe_parse_context {
unsigned int flags;
int offset;
int nested_level;
+ int prefix_byteoffs; /* The byte offset of the prefix field of typecast */
};
/* Each typecast consumes nested level. So the max number of typecast is 3. */
@@ -594,7 +595,9 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call,
C(EVENT_TOO_BIG, "Event too big (too many fields?)"), \
C(TYPECAST_NOT_EVENT, "Typecasts are only for eprobe fields"), \
C(TYPECAST_REQ_FIELD, "Typecast requires a field access"), \
- C(TOO_MANY_NESTED, "Too many nested typecasts/dereferences"),
+ C(TOO_MANY_NESTED, "Too many nested typecasts/dereferences"), \
+ C(TYPECAST_NOT_ALIGNED, "Typecast field option is not byte-aligned"), \
+ C(TYPECAST_BAD_ARROW, "Typecast field option does not support -> operator"),
#undef C
#define C(a, b) TP_ERR_##a
^ permalink raw reply related
* [PATCH v6 6/8] tracing/probes: Add $current variable support
From: Masami Hiramatsu (Google) @ 2026-06-20 15:18 UTC (permalink / raw)
To: Steven Rostedt, Mathieu Desnoyers
Cc: Jonathan Corbet, Shuah Khan, Masami Hiramatsu, linux-kernel,
linux-trace-kernel, linux-doc, linux-kselftest
In-Reply-To: <178196862271.560995.5255615288323003663.stgit@devnote2>
From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Since we can use the BTF to cast value to a structure pointer type,
it is useful to introduce "$current" special variable support to
fetcharg.
User can define a fetcharg to access current task_struct properties
using BTF info. e.g.
$current->cpus_ptr
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
---
Changes in v6:
- Rebased on dump fetcharg patch.
- Remove function name/eprobe requirement for $current.
Changes in v5:
- Use s32 for bof_find_btf_id().
Changes in v4:
- Add $current in README when CONFIG_HAVE_FUNCTION_ARG_ACCESS_API=y case.
- Fix to prohibit using $current in eprobes and address based kprobes.
Changes in v3:
- Remove $current support from eprobes (because eprobes is only for event)
- Prohibit uprobes to use $current.
Changes in v2:
- Support to parse $current in parse_btf_arg().
- If no typecast on $current, it automatically casted to task_struct.
- Check error case if $current follows something except for "-".
---
Documentation/trace/fprobetrace.rst | 1 +
Documentation/trace/kprobetrace.rst | 1 +
kernel/trace/trace.c | 4 ++--
kernel/trace/trace_probe.c | 40 ++++++++++++++++++++++++++++++++++-
kernel/trace/trace_probe.h | 1 +
kernel/trace/trace_probe_tmpl.h | 3 +++
6 files changed, 47 insertions(+), 3 deletions(-)
diff --git a/Documentation/trace/fprobetrace.rst b/Documentation/trace/fprobetrace.rst
index 290a9e6f7491..3392cab016b3 100644
--- a/Documentation/trace/fprobetrace.rst
+++ b/Documentation/trace/fprobetrace.rst
@@ -50,6 +50,7 @@ Synopsis of fprobe-events
$argN : Fetch the Nth function argument. (N >= 1) (\*2)
$retval : Fetch return value.(\*3)
$comm : Fetch current task comm.
+ $current : Fetch the address of the current task_struct.
+|-[u]OFFS(FETCHARG) : Fetch memory at FETCHARG +|- OFFS address.(\*4)(\*5)
\IMM : Store an immediate value to the argument.
NAME=FETCHARG : Set NAME as the argument name of FETCHARG.
diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst
index a62707e6a9f2..81e4fe38791d 100644
--- a/Documentation/trace/kprobetrace.rst
+++ b/Documentation/trace/kprobetrace.rst
@@ -53,6 +53,7 @@ Synopsis of kprobe_events
$argN : Fetch the Nth function argument. (N >= 1) (\*1)
$retval : Fetch return value.(\*2)
$comm : Fetch current task comm.
+ $current : Fetch the address of the current task_struct.
+|-[u]OFFS(FETCHARG) : Fetch memory at FETCHARG +|- OFFS address.(\*3)(\*4)
\IMM : Store an immediate value to the argument.
NAME=FETCHARG : Set NAME as the argument name of FETCHARG.
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 0e36af853199..7a5676524f1a 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -4323,13 +4323,13 @@ static const char readme_msg[] =
"\t args: <name>=fetcharg[:type]\n"
"\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
- "\t $stack<index>, $stack, $retval, $comm, $arg<N>,\n"
+ "\t $stack<index>, $stack, $retval, $comm, $arg<N>, $current\n"
#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
"\t [(structname[,field])]<argname>[->field[->field|.field...]],\n"
"\t [(structname[,field])](fetcharg)->field[->field|.field...],\n"
#endif
#else
- "\t $stack<index>, $stack, $retval, $comm,\n"
+ "\t $stack<index>, $stack, $retval, $comm, $current\n"
#endif
"\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
"\t kernel return probes support: $retval, $arg<N>, $comm\n"
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index 8c21c378fac7..1ddd0a804e39 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -700,7 +700,9 @@ static int parse_btf_arg(char *varname,
int i, is_ptr, ret;
u32 tid;
- if (!ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))
+ /* Note: field is not separated at this point, so check prefix. */
+ if (!str_has_prefix(varname, "$current") &&
+ !ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))
return -EINVAL;
is_ptr = split_next_field(varname, &field, ctx);
@@ -713,6 +715,24 @@ static int parse_btf_arg(char *varname,
return -EOPNOTSUPP;
}
+ if (!strcmp(varname, "$current")) {
+ code->op = FETCH_OP_CURRENT;
+ /* If no typecast is specified for $current, use task_struct by default */
+ if (!ctx->struct_btf) {
+ s32 ttid = bpf_find_btf_id("task_struct", BTF_KIND_STRUCT,
+ &ctx->struct_btf);
+
+ if (ttid < 0) {
+ trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
+ return -ENOENT;
+ }
+ /* btf_type_skip_modifier() requires u32 for type id. */
+ tid = ttid;
+ ctx->last_struct = btf_type_skip_modifiers(ctx->struct_btf, tid, &tid);
+ }
+ goto found;
+ }
+
if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
code->op = FETCH_OP_RETVAL;
/* Check whether the function return type is not void, even with typecast. */
@@ -1273,6 +1293,24 @@ static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
return 0;
}
+ /* $current returns the address of the current task_struct. */
+ if (str_has_prefix(arg, "current")) {
+ /* $current is only supported by kernel probe. */
+ if (!(ctx->flags & TPARG_FL_KERNEL)) {
+ err = TP_ERR_BAD_VAR;
+ goto inval;
+ }
+ arg += strlen("current");
+ if (*arg == '-' && IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS))
+ return parse_btf_arg(orig_arg, pcode, end, ctx);
+
+ if (*arg != '\0')
+ goto inval;
+
+ code->op = FETCH_OP_CURRENT;
+ return 0;
+ }
+
#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
len = str_has_prefix(arg, "arg");
if (len) {
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index bc3ac148a655..488d6790b5ef 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -92,6 +92,7 @@ typedef int (*print_type_func_t)(struct trace_seq *, void *, void *);
FETCH_OP(RETVAL, none) /* Return value */ \
FETCH_OP(IMM, imm) /* Immediate: .immediate */ \
FETCH_OP(COMM, none) /* Current comm */ \
+ FETCH_OP(CURRENT, none) /* Current task_struct address */\
FETCH_OP(ARG, param) /* Argument: .param = index */ \
FETCH_OP(FOFFS, imm) /* File offset: .immediate */ \
FETCH_OP(DATA, ptr) /* Allocated data: .data */ \
diff --git a/kernel/trace/trace_probe_tmpl.h b/kernel/trace/trace_probe_tmpl.h
index f39b37fcdb3b..f630930288d2 100644
--- a/kernel/trace/trace_probe_tmpl.h
+++ b/kernel/trace/trace_probe_tmpl.h
@@ -112,6 +112,9 @@ process_common_fetch_insn(struct fetch_insn *code, unsigned long *val)
case FETCH_OP_DATA:
*val = (unsigned long)code->data;
break;
+ case FETCH_OP_CURRENT:
+ *val = (unsigned long)current;
+ break;
default:
return -EILSEQ;
}
^ permalink raw reply related
* [PATCH v6 7/8] tracing/probes: Add this_cpu_read() and this_cpu_ptr() dereference method to fetcharg
From: Masami Hiramatsu (Google) @ 2026-06-20 15:18 UTC (permalink / raw)
To: Steven Rostedt, Mathieu Desnoyers
Cc: Jonathan Corbet, Shuah Khan, Masami Hiramatsu, linux-kernel,
linux-trace-kernel, linux-doc, linux-kselftest
In-Reply-To: <178196862271.560995.5255615288323003663.stgit@devnote2>
From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
When tracing the kernel local variables, sometimes we need to get the
CPU local variables. To access it, current simple dereference is not
enough.
Thus, introduce a special this_cpu_read() dereference to access per-cpu
variable for the current CPU (accessing other CPU variable may race with
updates on other CPUs). Also this_cpu_ptr() is for accessing per-cpu
pointer.
Those are working as same as the kernel percpu macro.
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
---
Changes in v6:
- Rebased on dump fetcharg patch.
- Fix to fetch static percpu variable with @SYM correctly.
Changes in v5:
- Simplify this_cpu_read() into +0(this_cpu_ptr()).
Changes in v3:
- Remove NULL check for percpu var because it is just an offset, could be 0.
- Simplify process_fetch_insn_bottom() code.
- If the last operation is this_cpu_read(), read only memory of the specific
size (of type).
Changes in v2:
- Drop +CPU/+PCPU and introduce this_cpu_read() and this_cpu_ptr().
- Support these method with BTF typecast.
- Just check the base address is NOT NULL instead of is_kernel_percpu_address().
---
Documentation/trace/eprobetrace.rst | 2
Documentation/trace/fprobetrace.rst | 2
Documentation/trace/kprobetrace.rst | 2
kernel/trace/trace.c | 1
kernel/trace/trace_probe.c | 143 ++++++++++++++++++++++++++---------
kernel/trace/trace_probe.h | 1
kernel/trace/trace_probe_tmpl.h | 22 ++++-
7 files changed, 129 insertions(+), 44 deletions(-)
diff --git a/Documentation/trace/eprobetrace.rst b/Documentation/trace/eprobetrace.rst
index 680e0af43d5d..279396951b34 100644
--- a/Documentation/trace/eprobetrace.rst
+++ b/Documentation/trace/eprobetrace.rst
@@ -39,6 +39,8 @@ Synopsis of eprobe_events
@SYM[+|-offs] : Fetch memory at SYM +|- offs (SYM should be a data symbol)
$comm : Fetch current task comm.
+|-[u]OFFS(FETCHARG) : Fetch memory at FETCHARG +|- OFFS address.(\*3)(\*4)
+ this_cpu_read(FETCHARG) : Read the value of the per-CPU variable FETCHARG on the current CPU.
+ this_cpu_ptr(FETCHARG) : Get the address of the per-CPU variable FETCHARG on the current CPU.
\IMM : Store an immediate value to the argument.
NAME=FETCHARG : Set NAME as the argument name of FETCHARG.
FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types
diff --git a/Documentation/trace/fprobetrace.rst b/Documentation/trace/fprobetrace.rst
index 3392cab016b3..3439bc9bd351 100644
--- a/Documentation/trace/fprobetrace.rst
+++ b/Documentation/trace/fprobetrace.rst
@@ -52,6 +52,8 @@ Synopsis of fprobe-events
$comm : Fetch current task comm.
$current : Fetch the address of the current task_struct.
+|-[u]OFFS(FETCHARG) : Fetch memory at FETCHARG +|- OFFS address.(\*4)(\*5)
+ this_cpu_read(FETCHARG) : Read the value of the per-CPU variable FETCHARG on the current CPU.
+ this_cpu_ptr(FETCHARG) : Get the address of the per-CPU variable FETCHARG on the current CPU.
\IMM : Store an immediate value to the argument.
NAME=FETCHARG : Set NAME as the argument name of FETCHARG.
FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types
diff --git a/Documentation/trace/kprobetrace.rst b/Documentation/trace/kprobetrace.rst
index 81e4fe38791d..9ae330eb0a52 100644
--- a/Documentation/trace/kprobetrace.rst
+++ b/Documentation/trace/kprobetrace.rst
@@ -55,6 +55,8 @@ Synopsis of kprobe_events
$comm : Fetch current task comm.
$current : Fetch the address of the current task_struct.
+|-[u]OFFS(FETCHARG) : Fetch memory at FETCHARG +|- OFFS address.(\*3)(\*4)
+ this_cpu_read(FETCHARG) : Read the value of the per-CPU variable FETCHARG on the current CPU.
+ this_cpu_ptr(FETCHARG) : Get the address of the per-CPU variable FETCHARG on the current CPU.
\IMM : Store an immediate value to the argument.
NAME=FETCHARG : Set NAME as the argument name of FETCHARG.
FETCHARG:TYPE : Set TYPE as the type of FETCHARG. Currently, basic types
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 7a5676524f1a..d4121acc2938 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -4332,6 +4332,7 @@ static const char readme_msg[] =
"\t $stack<index>, $stack, $retval, $comm, $current\n"
#endif
"\t +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
+ "\t this_cpu_read(<fetcharg>), this_cpu_ptr(<fetcharg>)\n"
"\t kernel return probes support: $retval, $arg<N>, $comm\n"
"\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, char, string, symbol,\n"
"\t b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c
index 1ddd0a804e39..a64edb0c6baa 100644
--- a/kernel/trace/trace_probe.c
+++ b/kernel/trace/trace_probe.c
@@ -349,6 +349,100 @@ static int parse_trace_event(char *arg, struct fetch_insn *code,
return -EINVAL;
}
+/* this_cpu_* parser */
+#define THIS_CPU_PTR_PREFIX "this_cpu_ptr("
+#define THIS_CPU_READ_PREFIX "this_cpu_read("
+#define THIS_CPU_PTR_LEN (sizeof(THIS_CPU_PTR_PREFIX) - 1)
+#define THIS_CPU_READ_LEN (sizeof(THIS_CPU_READ_PREFIX) - 1)
+
+static int
+parse_probe_arg(char *arg, const struct fetch_type *type,
+ struct fetch_insn **pcode, struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx);
+
+/* handle dereference nested call */
+static inline int handle_dereference(char *arg, struct fetch_insn **pcode,
+ struct fetch_insn *end, struct traceprobe_parse_context *ctx,
+ int deref, long offset)
+{
+ const struct fetch_type *type = find_fetch_type(NULL, ctx->flags);
+ struct fetch_insn *code = *pcode;
+ int cur_offs = ctx->offset;
+ char *tmp;
+ int ret;
+
+ tmp = strrchr(arg, ')');
+ if (!tmp) {
+ trace_probe_log_err(ctx->offset + strlen(arg),
+ DEREF_OPEN_BRACE);
+ return -EINVAL;
+ }
+
+ *tmp = '\0';
+ ret = parse_probe_arg(arg, type, &code, end, ctx);
+ if (ret)
+ return ret;
+ ctx->offset = cur_offs;
+ if (code->op == FETCH_OP_COMM || code->op == FETCH_OP_DATA) {
+ trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
+ return -EINVAL;
+ }
+
+ /*
+ * this_cpu_ptr(@SYM) does not use SYM value, but use SYM address.
+ * So we overwrite the last FETCH_OP_DEREF with FETCH_OP_CPU_PTR.
+ */
+ if (!(deref == FETCH_OP_CPU_PTR && *arg == '@')) {
+ code++;
+ if (code == end) {
+ trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
+ return -EINVAL;
+ }
+ *pcode = code;
+ }
+
+ code->op = deref;
+ code->offset = offset;
+ /* Reset the last type if used */
+ ctx->last_type = NULL;
+ return 0;
+}
+
+static int parse_this_cpu(char *arg, struct fetch_insn **pcode,
+ struct fetch_insn *end,
+ struct traceprobe_parse_context *ctx)
+{
+ struct fetch_insn *code;
+ bool is_ptr = false;
+ int ret;
+
+ if (str_has_prefix(arg, THIS_CPU_PTR_PREFIX)) {
+ arg += THIS_CPU_PTR_LEN;
+ ctx->offset += THIS_CPU_PTR_LEN;
+ is_ptr = true;
+ } else if (str_has_prefix(arg, THIS_CPU_READ_PREFIX)) {
+ arg += THIS_CPU_READ_LEN;
+ ctx->offset += THIS_CPU_READ_LEN;
+ } else
+ return -EINVAL;
+
+ ret = handle_dereference(arg, pcode, end, ctx, FETCH_OP_CPU_PTR, 0);
+ if (ret || is_ptr)
+ return ret;
+
+ /* this_cpu_read(VAR) -> +0(this_cpu_ptr(VAR)) */
+ code = *pcode;
+ code++;
+ if (code == end) {
+ trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
+ return -EINVAL;
+ }
+ code->op = FETCH_OP_DEREF;
+ code->offset = 0;
+ *pcode = code;
+ return 0;
+}
+
#ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
static u32 btf_type_int(const struct btf_type *t)
@@ -925,11 +1019,6 @@ static char *find_matched_close_paren(char *s)
return NULL;
}
-static int
-parse_probe_arg(char *arg, const struct fetch_type *type,
- struct fetch_insn **pcode, struct fetch_insn *end,
- struct traceprobe_parse_context *ctx);
-
static int handle_typecast(char *arg, struct fetch_insn **pcode,
struct fetch_insn *end,
struct traceprobe_parse_context *ctx)
@@ -982,7 +1071,9 @@ static int handle_typecast(char *arg, struct fetch_insn **pcode,
/* Skip '(' */
ctx->offset += 1;
tmp++;
- } else if (*tmp == '+' || *tmp == '-') {
+ } else if (*tmp == '+' || *tmp == '-' ||
+ str_has_prefix(tmp, THIS_CPU_PTR_PREFIX) ||
+ str_has_prefix(tmp, THIS_CPU_READ_PREFIX)) {
/* Dereference can have another field access inside it. */
char *open = strchr(tmp + 1, '(');
@@ -1486,36 +1577,9 @@ parse_probe_arg(char *arg, const struct fetch_type *type,
}
ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
arg = tmp + 1;
- tmp = strrchr(arg, ')');
- if (!tmp) {
- trace_probe_log_err(ctx->offset + strlen(arg),
- DEREF_OPEN_BRACE);
- return -EINVAL;
- } else {
- const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags);
- int cur_offs = ctx->offset;
-
- *tmp = '\0';
- ret = parse_probe_arg(arg, t2, &code, end, ctx);
- if (ret)
- break;
- ctx->offset = cur_offs;
- if (code->op == FETCH_OP_COMM ||
- code->op == FETCH_OP_DATA) {
- trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
- return -EINVAL;
- }
- if (++code == end) {
- trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
- return -EINVAL;
- }
- *pcode = code;
-
- code->op = deref;
- code->offset = offset;
- /* Reset the last type if used */
- ctx->last_type = NULL;
- }
+ ret = handle_dereference(arg, pcode, end, ctx, deref, offset);
+ if (ret < 0)
+ return ret;
break;
case '\\': /* Immediate value */
if (arg[1] == '"') { /* Immediate string */
@@ -1536,15 +1600,18 @@ parse_probe_arg(char *arg, const struct fetch_type *type,
ret = handle_typecast(arg, pcode, end, ctx);
break;
default:
- if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */
+ if (str_has_prefix(arg, THIS_CPU_PTR_PREFIX) ||
+ str_has_prefix(arg, THIS_CPU_READ_PREFIX)) {
+ ret = parse_this_cpu(arg, pcode, end, ctx);
+ } else if (isalpha(arg[0]) || arg[0] == '_') { /* BTF variable */
if (!tparg_is_function_entry(ctx->flags) &&
!tparg_is_function_return(ctx->flags)) {
trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
return -EINVAL;
}
ret = parse_btf_arg(arg, pcode, end, ctx);
- break;
}
+ break;
}
if (!ret && code->op == FETCH_OP_NOP) {
/* Parsed, but do not find fetch method */
diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h
index 488d6790b5ef..e79e019b922d 100644
--- a/kernel/trace/trace_probe.h
+++ b/kernel/trace/trace_probe.h
@@ -101,6 +101,7 @@ typedef int (*print_type_func_t)(struct trace_seq *, void *, void *);
/* Stage 2 (dereference) ops */ \
FETCH_OP(DEREF, offset) /* Dereference: .offset */ \
FETCH_OP(UDEREF, offset) /* User-space dereference: .offset */\
+ FETCH_OP(CPU_PTR, none) /* Per-CPU pointer: .offset */ \
/* Stage 3 (store) ops */ \
FETCH_OP(ST_RAW, store) /* Raw value: .size */ \
FETCH_OP(ST_MEM, store) /* Memory: .offset, .size */ \
diff --git a/kernel/trace/trace_probe_tmpl.h b/kernel/trace/trace_probe_tmpl.h
index f630930288d2..9265b03cf19d 100644
--- a/kernel/trace/trace_probe_tmpl.h
+++ b/kernel/trace/trace_probe_tmpl.h
@@ -129,25 +129,35 @@ process_fetch_insn_bottom(struct fetch_insn *code, unsigned long val,
struct fetch_insn *s3 = NULL;
int total = 0, ret = 0, i = 0;
u32 loc = 0;
- unsigned long lval = val;
+ unsigned long lval, llval = val;
stage2:
/* 2nd stage: dereference memory if needed */
do {
- if (code->op == FETCH_OP_DEREF) {
- lval = val;
+ lval = val;
+ switch (code->op) {
+ case FETCH_OP_DEREF:
ret = probe_mem_read(&val, (void *)val + code->offset,
sizeof(val));
- } else if (code->op == FETCH_OP_UDEREF) {
- lval = val;
+ break;
+ case FETCH_OP_UDEREF:
ret = probe_mem_read_user(&val,
(void *)val + code->offset, sizeof(val));
- } else
break;
+ case FETCH_OP_CPU_PTR:
+ val = (unsigned long)this_cpu_ptr((void __percpu *)val);
+ ret = 0;
+ break;
+ default:
+ lval = llval;
+ goto out;
+ }
if (ret)
return ret;
+ llval = lval;
code++;
} while (1);
+out:
s3 = code;
stage3:
^ permalink raw reply related
* [PATCH v6 8/8] tracing/probes: Add a new testcase for BTF typecasts
From: Masami Hiramatsu (Google) @ 2026-06-20 15:18 UTC (permalink / raw)
To: Steven Rostedt, Mathieu Desnoyers
Cc: Jonathan Corbet, Shuah Khan, Masami Hiramatsu, linux-kernel,
linux-trace-kernel, linux-doc, linux-kselftest
In-Reply-To: <178196862271.560995.5255615288323003663.stgit@devnote2>
From: Masami Hiramatsu (Google) <mhiramat@kernel.org>
With the introduction of container_of-style BTF typecasting and
per-CPU variable access support in trace probes, we need a way to
verify their functionality and prevent regressions.
Add a new ftrace kselftest and update the trace event sample module
to test and validate these features.
Specifically, update the trace-events-sample module to set up a
periodic timer whose callback accesses a per-CPU counter. Introduce
a new sample trace event, foo_timer_fn, to trace this callback
and log the current counter value.
Then, add a new test case, btf_probe_event.tc, which defines a
dynamic probe on the timer callback. The probe uses BTF typecasting
to recover the parent structure from the timer argument and
this_cpu_read() to fetch the per-CPU counter. The test verifies
the integrity of the implementation by ensuring the values
recorded by the dynamic probe match those from the static tracepoint.
Assisted-by: Antigravity:gemini-3.5-flash
Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
---
Changes in v6:
- Update testcase according to changes.
Changes in v5:
- Add more syntax test cases.
Changes in v4:
- Fix uprobe $current test.
Changes in v3:
- Add syntax test case.
- Update testcase to use this_cpu_read()
Changes in v2:
- Use timer_shutdown_sync() instead of timer_delete_sync() for teardown.
---
samples/trace_events/trace-events-sample.c | 40 +++++++++++++++-
samples/trace_events/trace-events-sample.h | 34 ++++++++++++-
.../ftrace/test.d/dynevent/btf_probe_event.tc | 51 ++++++++++++++++++++
.../ftrace/test.d/dynevent/fprobe_syntax_errors.tc | 11 ++++
.../ftrace/test.d/kprobe/kprobe_syntax_errors.tc | 11 ++++
.../ftrace/test.d/kprobe/uprobe_syntax_errors.tc | 5 ++
6 files changed, 147 insertions(+), 5 deletions(-)
create mode 100644 tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc
diff --git a/samples/trace_events/trace-events-sample.c b/samples/trace_events/trace-events-sample.c
index 0b7a6efdb247..ca5d98c360cb 100644
--- a/samples/trace_events/trace-events-sample.c
+++ b/samples/trace_events/trace-events-sample.c
@@ -94,6 +94,20 @@ static int simple_thread_fn(void *arg)
static DEFINE_MUTEX(thread_mutex);
static int simple_thread_cnt;
+static struct foo_timer_data *foo_timer_data;
+
+static void sample_timer_cb(struct timer_list *t)
+{
+ struct foo_timer_data *data = container_of(t, struct foo_timer_data, timer);
+
+ get_cpu();
+ trace_foo_timer_fn(data);
+ (*this_cpu_ptr(data->counter))++;
+ put_cpu();
+
+ mod_timer(t, jiffies + HZ);
+}
+
int foo_bar_reg(void)
{
mutex_lock(&thread_mutex);
@@ -132,9 +146,27 @@ void foo_bar_unreg(void)
static int __init trace_event_init(void)
{
+ foo_timer_data = kzalloc_obj(*foo_timer_data, GFP_KERNEL);
+ if (!foo_timer_data)
+ return -ENOMEM;
+
+ foo_timer_data->name = "sample_timer_counter";
+ foo_timer_data->counter = alloc_percpu(int);
+ if (!foo_timer_data->counter) {
+ kfree(foo_timer_data);
+ return -ENOMEM;
+ }
+
+ timer_setup(&foo_timer_data->timer, sample_timer_cb, 0);
+ mod_timer(&foo_timer_data->timer, jiffies + HZ);
+
simple_tsk = kthread_run(simple_thread, NULL, "event-sample");
- if (IS_ERR(simple_tsk))
- return -1;
+ if (IS_ERR(simple_tsk)) {
+ timer_shutdown_sync(&foo_timer_data->timer);
+ free_percpu(foo_timer_data->counter);
+ kfree(foo_timer_data);
+ return PTR_ERR(simple_tsk);
+ }
return 0;
}
@@ -147,6 +179,10 @@ static void __exit trace_event_exit(void)
kthread_stop(simple_tsk_fn);
simple_tsk_fn = NULL;
mutex_unlock(&thread_mutex);
+
+ timer_shutdown_sync(&foo_timer_data->timer);
+ free_percpu(foo_timer_data->counter);
+ kfree(foo_timer_data);
}
module_init(trace_event_init);
diff --git a/samples/trace_events/trace-events-sample.h b/samples/trace_events/trace-events-sample.h
index 1a05fc153353..816848a456a2 100644
--- a/samples/trace_events/trace-events-sample.h
+++ b/samples/trace_events/trace-events-sample.h
@@ -247,12 +247,14 @@
*/
/*
- * It is OK to have helper functions in the file, but they need to be protected
- * from being defined more than once. Remember, this file gets included more
- * than once.
+ * It is OK to have helper functions and data structures in the file, but they
+ * need to be protected from being defined more than once. Remember, this file
+ * gets included more than once.
*/
#ifndef __TRACE_EVENT_SAMPLE_HELPER_FUNCTIONS
#define __TRACE_EVENT_SAMPLE_HELPER_FUNCTIONS
+#include <linux/timer.h>
+
static inline int __length_of(const int *list)
{
int i;
@@ -270,6 +272,13 @@ enum {
TRACE_SAMPLE_BAR = 4,
TRACE_SAMPLE_ZOO = 8,
};
+
+struct foo_timer_data {
+ const char *name;
+ struct timer_list timer;
+ int __percpu *counter;
+};
+
#endif
/*
@@ -595,6 +604,25 @@ TRACE_EVENT(foo_rel_loc,
__get_rel_bitmask(bitmask),
__get_rel_cpumask(cpumask))
);
+
+TRACE_EVENT(foo_timer_fn,
+
+ TP_PROTO(struct foo_timer_data *data),
+
+ TP_ARGS(data),
+
+ TP_STRUCT__entry(
+ __string( name, data->name )
+ __field( int, count )
+ ),
+
+ TP_fast_assign(
+ __assign_str(name);
+ __entry->count = *this_cpu_ptr(data->counter);
+ ),
+
+ TP_printk("name=%s count=%d", __get_str(name), __entry->count)
+);
#endif
/***** NOTICE! The #if protection ends here. *****/
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc b/tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc
new file mode 100644
index 000000000000..96791e120b7d
--- /dev/null
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/btf_probe_event.tc
@@ -0,0 +1,51 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+# description: BTF event with typecast and percpu access
+# requires: dynamic_events "this_cpu_read(<fetcharg>)":README "[(structname[,field])]<argname>[->field[->field|.field...]]":README
+
+# Check if the sample module is loaded
+if ! lsmod | grep -q trace_events_sample; then
+ modprobe trace-events-sample || exit_unsupported
+fi
+
+echo 0 > events/enable
+echo > dynamic_events
+
+# The sample_timer_cb(struct timer_list *t) is called.
+# We want to check (STRUCT,FIELD)VAR typecast and this_cpu_read() access.
+# (foo_timer_data,timer)t converts t to struct foo_timer_data * using container_of.
+# data->counter is a per-cpu pointer to int.
+# this_cpu_read(data->counter) should give the value of the counter.
+
+echo 'f:mysample/myevent sample_timer_cb name=(foo_timer_data,timer)t->name:string count=this_cpu_read((foo_timer_data,timer)t->counter)' >> dynamic_events
+
+echo 1 > events/mysample/myevent/enable
+echo 1 > events/sample-trace/foo_timer_fn/enable
+
+sleep 2
+
+echo 0 > events/mysample/myevent/enable
+echo 0 > events/sample-trace/foo_timer_fn/enable
+
+# Compare the values.
+MATCH=0
+while read line; do
+ if echo $line | grep -q "foo_timer_fn:"; then
+ NAME=`echo $line | sed 's/.*name=\([^ ]*\) .*/\1/'`
+ COUNT=`echo $line | sed 's/.*count=\([^ ]*\).*/\1/'`
+ if grep -q "myevent:.*name=\"${NAME}\" count=$COUNT" trace; then
+ MATCH=$((MATCH+1))
+ fi
+ fi
+done < trace
+
+if [ $MATCH -eq 0 ]; then
+ echo "No matching events found"
+ exit_fail
+fi
+
+# Clean up
+echo 0 > events/mysample/myevent/enable
+echo 0 > events/sample-trace/foo_timer_fn/enable
+echo > dynamic_events
+clear_trace
diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc
index fee479295e2f..e111d426a984 100644
--- a/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc
+++ b/tools/testing/selftests/ftrace/test.d/dynevent/fprobe_syntax_errors.tc
@@ -112,6 +112,17 @@ check_error 'f vfs_read%return $retval->^foo' # NO_PTR_STRCT
check_error 'f vfs_read file->^foo' # NO_BTF_FIELD
check_error 'f vfs_read file^-.foo' # BAD_HYPHEN
check_error 'f vfs_read ^file:string' # BAD_TYPE4STR
+if grep -qF "[(structname" README ; then
+check_error 'f vfs_read arg1=(task_struct)file^' # TYPECAST_REQ_FIELD
+check_error 'f vfs_read arg1=(a)((b)((c)(^(d)file->d)->c)->b)->a' # TOO_MANY_NESTED
+check_error 'f vfs_read arg1=(task_struct,^in_execve)file->comm' # TYPECAST_NOT_ALIGNED
+check_error 'f vfs_read arg1=(task_struct,^foo_bar)file->pid' # NO_BTF_FIELD
+check_error 'f vfs_read arg1=(^task_struct1234)file->pid' # NO_PTR_STRCT
+check_error 'f vfs_read arg1=(task_struct,se^->group_node)file->comm' # TYPECAST_BAD_ARROW
+check_error 'f vfs_read arg1=(task_struct,^->pid)file->comm' # NO_BTF_FIELD
+check_error 'f vfs_read arg1=(task_struct,^.pid)file->comm' # NO_BTF_FIELD
+check_error 'f vfs_read arg1=(task_struct,^.)file->comm' # NO_BTF_FIELD
+fi
fi
else
diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc
index 8f1c58f0c239..626adeb2e840 100644
--- a/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc
+++ b/tools/testing/selftests/ftrace/test.d/kprobe/kprobe_syntax_errors.tc
@@ -115,6 +115,17 @@ check_error 'p vfs_read+20 ^$arg*' # NOFENTRY_ARGS
check_error 'p vfs_read ^hoge' # NO_BTFARG
check_error 'p kfree ^$arg10' # NO_BTFARG (exceed the number of parameters)
check_error 'r kfree ^$retval' # NO_RETVAL
+if grep -qF "[(structname" README ; then
+check_error 'p vfs_read arg1=(task_struct)file^' # TYPECAST_REQ_FIELD
+check_error 'p vfs_read arg1=(a)((b)((c)(^(d)file->d)->c)->b)->a' # TOO_MANY_NESTED
+check_error 'p vfs_read arg1=(task_struct,^in_execve)file->comm' # TYPECAST_NOT_ALIGNED
+check_error 'p vfs_read arg1=(task_struct,^foo_bar)file->pid' # NO_BTF_FIELD
+check_error 'p vfs_read arg1=(^task_struct1234)file->pid' # NO_PTR_STRCT
+check_error 'p vfs_read arg1=(task_struct,se^->group_node)file->comm' # TYPECAST_BAD_ARROW
+check_error 'p vfs_read arg1=(task_struct,^->pid)file->comm' # NO_BTF_FIELD
+check_error 'p vfs_read arg1=(task_struct,^.pid)file->comm' # NO_BTF_FIELD
+check_error 'p vfs_read arg1=(task_struct,^.)file->comm' # NO_BTF_FIELD
+fi
else
check_error 'p vfs_read ^$arg*' # NOSUP_BTFARG
fi
diff --git a/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc b/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc
index c817158b99db..e12dc967ec76 100644
--- a/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc
+++ b/tools/testing/selftests/ftrace/test.d/kprobe/uprobe_syntax_errors.tc
@@ -28,4 +28,9 @@ if grep -q ".*symstr.*" README; then
check_error 'p /bin/sh:10 $stack0:^symstr' # BAD_TYPE
fi
+# $current is not supported by uprobe
+if grep -q "\$current.*" README; then
+check_error 'p /bin/sh:10 ^$current:u8' # BAD_VAR
+fi
+
exit 0
^ permalink raw reply related
* Re: [PATCH] docs: ipmi: Fix path of the "hotmod" module parameter
From: Corey Minyard @ 2026-06-20 15:23 UTC (permalink / raw)
To: Zenghui Yu; +Cc: openipmi-developer, linux-doc, linux-kernel, corbet, skhan
In-Reply-To: <20260620122747.7902-1-zenghui.yu@linux.dev>
On Sat, Jun 20, 2026 at 08:27:47PM +0800, Zenghui Yu wrote:
> The correct path of the "hotmod" module parameter should be
> /sys/module/ipmi_si/parameters/hotmod. Fix it.
Thanks, it's in my queue.
-corey
>
> Signed-off-by: Zenghui Yu <zenghui.yu@linux.dev>
> ---
> Documentation/driver-api/ipmi.rst | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/Documentation/driver-api/ipmi.rst b/Documentation/driver-api/ipmi.rst
> index f52ab2df2569..d08cee98e34a 100644
> --- a/Documentation/driver-api/ipmi.rst
> +++ b/Documentation/driver-api/ipmi.rst
> @@ -495,7 +495,7 @@ tuned to the user's desired performance.
>
> The driver supports a hot add and remove of interfaces. This way,
> interfaces can be added or removed after the kernel is up and running.
> -This is done using /sys/modules/ipmi_si/parameters/hotmod, which is a
> +This is done using /sys/module/ipmi_si/parameters/hotmod, which is a
> write-only parameter. You write a string to this interface. The string
> has the format::
>
> --
> 2.53.0
>
^ permalink raw reply
* [PATCH v3 1/3] dmaengine: dw-edma-pcie: Discover endpoint DMA metadata
From: Koichiro Den @ 2026-06-20 17:08 UTC (permalink / raw)
To: Manivannan Sadhasivam, Krzysztof Wilczyński,
Kishon Vijay Abraham I, Bjorn Helgaas, Jonathan Corbet,
Shuah Khan, Vinod Koul, Frank Li, Arnd Bergmann, Damien Le Moal,
Niklas Cassel
Cc: Marek Vasut, Yoshihiro Shimoda, linux-pci, linux-doc,
linux-kernel, dmaengine
In-Reply-To: <20260620170844.3757241-1-den@valinux.co.jp>
Teach dw-edma-pcie to discover a PCI endpoint DMA function from
BAR-resident metadata. The metadata supplies the DMA register window,
channel counts, descriptor windows, optional auxiliary windows, and
endpoint-local descriptor and auxiliary addresses. Accept DesignWare
eDMA unroll, HDMA compatible, and HDMA native linked-list layouts.
Endpoint-provided DMA channels use raw slave addresses because the host
programs transfers against endpoint physical addresses, not PCI BAR
addresses. The host-side dw-edma-pcie instance is remote-routed by
default, so delegated channels report completions through IMWr/MSI.
Endpoint DMA metadata currently has no static PCI ID. Let an explicit
driver_override bind use the generic endpoint DMA metadata parser, but
do not treat arbitrary dynamic IDs without driver data as endpoint DMA
devices.
The endpoint polls HOST_REQ at a low idle rate before programming DMA
window submaps and setting READY. Let the host wait for several endpoint
poll periods before treating the READY handshake as timed out.
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes in v3:
- Select endpoint DMA match data before copying DMA data (Sashiko).
- Require driver_override for the generic endpoint DMA fallback
(Sashiko).
- Accept HDMA native linked-list endpoint DMA metadata.
drivers/dma/dw-edma/dw-edma-pcie.c | 380 ++++++++++++++++++++++++++++-
1 file changed, 378 insertions(+), 2 deletions(-)
diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c
index 1e75fefae9b8..2a56ee19d4cb 100644
--- a/drivers/dma/dw-edma/dw-edma-pcie.c
+++ b/drivers/dma/dw-edma/dw-edma-pcie.c
@@ -11,9 +11,13 @@
#include <linux/pci.h>
#include <linux/device.h>
#include <linux/dma/edma.h>
+#include <linux/iopoll.h>
#include <linux/pci-epf.h>
#include <linux/msi.h>
#include <linux/bitfield.h>
+#include <linux/io.h>
+#include <linux/overflow.h>
+#include <linux/pci-ep-dma.h>
#include <linux/sizes.h>
#include "dw-edma-core.h"
@@ -45,6 +49,9 @@
#define DW_PCIE_XILINX_MDB_DT_OFF_GAP 0x100000
#define DW_PCIE_XILINX_MDB_DT_SIZE 0x800
+#define DW_PCIE_EP_DMA_READY_POLL_US 1000
+#define DW_PCIE_EP_DMA_READY_TIMEOUT_US 2000000
+
#define DW_BLOCK(a, b, c) \
{ \
.bar = a, \
@@ -94,6 +101,12 @@ struct dw_edma_pcie_match_data {
#define DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF BIT(0)
#define DW_EDMA_PCIE_F_REG_OFFSET BIT(1)
+struct dw_edma_pcie_ep_dma_view {
+ struct pci_dev *pdev;
+ void __iomem *base;
+ resource_size_t limit;
+};
+
static const struct dw_edma_pcie_data snps_edda_data = {
/* eDMA registers location */
.rg.bar = BAR_0,
@@ -158,6 +171,13 @@ static const struct dw_edma_pcie_data xilinx_cpm6_dma_data = {
.rd_ch_cnt = 8,
};
+static const struct dw_edma_pcie_data ep_dma_data = {
+ .mf = EDMA_MF_EDMA_UNROLL,
+ .irqs = EDMA_MAX_WR_CH + EDMA_MAX_RD_CH,
+ .wr_ch_cnt = EDMA_MAX_WR_CH,
+ .rd_ch_cnt = EDMA_MAX_RD_CH,
+};
+
static void dw_edma_set_chan_region_offset(struct dw_edma_pcie_data *pdata,
enum pci_barno bar, off_t start_off,
off_t ll_off_gap, size_t ll_size,
@@ -227,6 +247,86 @@ static const struct dw_edma_plat_ops dw_edma_pcie_plat_ops = {
.pci_address = dw_edma_pcie_address,
};
+static const struct dw_edma_plat_ops dw_edma_pcie_raw_addr_plat_ops = {
+ .irq_vector = dw_edma_pcie_irq_vector,
+};
+
+static bool dw_edma_pcie_valid_bar(enum pci_barno bar)
+{
+ return bar >= BAR_0 && bar <= BAR_5;
+}
+
+static bool dw_edma_pcie_valid_bar_range(struct pci_dev *pdev,
+ enum pci_barno bar, u64 off,
+ size_t sz)
+{
+ resource_size_t bar_len;
+
+ if (!dw_edma_pcie_valid_bar(bar) || !sz)
+ return false;
+
+ bar_len = pci_resource_len(pdev, bar);
+
+ return off <= bar_len && sz <= bar_len - off;
+}
+
+static bool dw_edma_pcie_valid_block(struct pci_dev *pdev,
+ const struct dw_edma_block *block)
+{
+ return dw_edma_pcie_valid_bar_range(pdev, block->bar, block->off,
+ block->sz);
+}
+
+static bool dw_edma_pcie_ep_dma_bar_scannable(struct pci_dev *pdev,
+ enum pci_barno bar)
+{
+ unsigned long flags = pci_resource_flags(pdev, bar);
+
+ if (!(flags & IORESOURCE_MEM))
+ return false;
+
+ if (flags & (IORESOURCE_UNSET | IORESOURCE_DISABLED))
+ return false;
+
+ return pci_resource_len(pdev, bar) >= PCI_EP_DMA_METADATA_HDR_LEN;
+}
+
+static u32 dw_edma_pcie_ep_dma_readl(struct dw_edma_pcie_ep_dma_view *view,
+ u16 off)
+{
+ return readl(view->base + off);
+}
+
+static void dw_edma_pcie_ep_dma_writel(struct dw_edma_pcie_ep_dma_view *view,
+ u16 off, u32 val)
+{
+ writel(val, view->base + off);
+}
+
+static u64 dw_edma_pcie_ep_dma_read64(struct dw_edma_pcie_ep_dma_view *view,
+ u16 lo, u16 hi)
+{
+ u64 val;
+
+ val = dw_edma_pcie_ep_dma_readl(view, hi);
+
+ return (val << 32) | dw_edma_pcie_ep_dma_readl(view, lo);
+}
+
+static int dw_edma_pcie_ep_dma_read_off(struct dw_edma_pcie_ep_dma_view *view,
+ u16 lo, u16 hi, off_t *off)
+{
+ u64 val;
+
+ val = dw_edma_pcie_ep_dma_read64(view, lo, hi);
+ if (val > type_max(*off))
+ return -EINVAL;
+
+ *off = val;
+
+ return 0;
+}
+
static void dw_edma_pcie_get_synopsys_dma_data(struct pci_dev *pdev,
struct dw_edma_pcie_data *pdata)
{
@@ -328,6 +428,265 @@ static void dw_edma_pcie_get_xilinx_dma_data(struct pci_dev *pdev,
pdata->devmem_phys_off = off;
}
+static int
+dw_edma_pcie_parse_ep_dma_ch_table(struct dw_edma_pcie_ep_dma_view *view,
+ struct dw_edma_pcie_data *pdata,
+ u16 table_off, u16 entry_size, u16 ch_cnt,
+ bool write)
+{
+ struct dw_edma_block *desc_blocks = write ? pdata->ll_wr : pdata->ll_rd;
+ struct dw_edma_block *data_blocks = write ? pdata->dt_wr : pdata->dt_rd;
+ u32 ctrl;
+ u16 i;
+ int ret;
+
+ for (i = 0; i < ch_cnt; i++) {
+ struct dw_edma_block *desc_block = &desc_blocks[i];
+ struct dw_edma_block *data_block = &data_blocks[i];
+ u16 off = table_off + i * entry_size;
+ u16 field, lo, hi;
+
+ field = off + PCI_EP_DMA_METADATA_CH_CTRL;
+ ctrl = dw_edma_pcie_ep_dma_readl(view, field);
+ if (FIELD_GET(PCI_EP_DMA_METADATA_CH_CTRL_HW_CH, ctrl) != i)
+ return -EOPNOTSUPP;
+
+ desc_block->bar =
+ FIELD_GET(PCI_EP_DMA_METADATA_CH_CTRL_DESC_BAR, ctrl);
+ lo = off + PCI_EP_DMA_METADATA_CH_DESC_OFF_LO;
+ hi = off + PCI_EP_DMA_METADATA_CH_DESC_OFF_HI;
+ ret = dw_edma_pcie_ep_dma_read_off(view, lo, hi,
+ &desc_block->off);
+ if (ret)
+ return ret;
+ field = off + PCI_EP_DMA_METADATA_CH_DESC_SIZE;
+ desc_block->sz = dw_edma_pcie_ep_dma_readl(view, field);
+ lo = off + PCI_EP_DMA_METADATA_CH_DESC_ADDR_LO;
+ hi = off + PCI_EP_DMA_METADATA_CH_DESC_ADDR_HI;
+ desc_block->paddr =
+ dw_edma_pcie_ep_dma_read64(view, lo, hi);
+ desc_block->paddr_valid = true;
+ if (!dw_edma_pcie_valid_block(view->pdev, desc_block))
+ return -EINVAL;
+
+ *data_block = (struct dw_edma_block) { .bar = NO_BAR };
+ if (!(ctrl & PCI_EP_DMA_METADATA_CH_CTRL_AUX_VALID))
+ continue;
+
+ data_block->bar =
+ FIELD_GET(PCI_EP_DMA_METADATA_CH_CTRL_AUX_BAR, ctrl);
+ lo = off + PCI_EP_DMA_METADATA_CH_AUX_OFF_LO;
+ hi = off + PCI_EP_DMA_METADATA_CH_AUX_OFF_HI;
+ ret = dw_edma_pcie_ep_dma_read_off(view, lo, hi,
+ &data_block->off);
+ if (ret)
+ return ret;
+ field = off + PCI_EP_DMA_METADATA_CH_AUX_SIZE;
+ data_block->sz = dw_edma_pcie_ep_dma_readl(view, field);
+ lo = off + PCI_EP_DMA_METADATA_CH_AUX_ADDR_LO;
+ hi = off + PCI_EP_DMA_METADATA_CH_AUX_ADDR_HI;
+ data_block->paddr =
+ dw_edma_pcie_ep_dma_read64(view, lo, hi);
+ data_block->paddr_valid = true;
+ if (!dw_edma_pcie_valid_block(view->pdev, data_block))
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int
+dw_edma_pcie_ep_dma_wait_ready(struct dw_edma_pcie_ep_dma_view *view)
+{
+ u32 val;
+
+ return read_poll_timeout(dw_edma_pcie_ep_dma_readl, val,
+ val & PCI_EP_DMA_METADATA_CTRL_READY,
+ DW_PCIE_EP_DMA_READY_POLL_US,
+ DW_PCIE_EP_DMA_READY_TIMEOUT_US, false,
+ view, PCI_EP_DMA_METADATA_CTRL);
+}
+
+static int
+dw_edma_pcie_validate_ep_dma_metadata(struct dw_edma_pcie_ep_dma_view *view,
+ u32 *metadata_ctrl, u8 *reg_layout_data)
+{
+ size_t table_size, table_end;
+ enum pci_barno reg_bar;
+ u16 len, entry_size;
+ u16 wr_ch_cnt, rd_ch_cnt;
+ u8 layout, layout_data;
+ u32 val;
+
+ val = dw_edma_pcie_ep_dma_readl(view, 0);
+ if (val != PCI_EP_DMA_METADATA_MAGIC)
+ return -ENODEV;
+
+ val = dw_edma_pcie_ep_dma_readl(view, PCI_EP_DMA_METADATA_HDR);
+ if (FIELD_GET(PCI_EP_DMA_METADATA_HDR_REV, val) !=
+ PCI_EP_DMA_METADATA_REV)
+ return -EINVAL;
+
+ len = FIELD_GET(PCI_EP_DMA_METADATA_HDR_LEN_FIELD, val);
+ if (len < PCI_EP_DMA_METADATA_HDR_LEN)
+ return -EINVAL;
+ if (len > view->limit)
+ return -EINVAL;
+
+ val = dw_edma_pcie_ep_dma_readl(view, PCI_EP_DMA_METADATA_REG_LAYOUT);
+ layout = FIELD_GET(PCI_EP_DMA_METADATA_REG_LAYOUT_ID, val);
+ if (layout != PCI_EP_DMA_METADATA_REG_LAYOUT_DW_EDMA)
+ return -EOPNOTSUPP;
+
+ layout_data = FIELD_GET(PCI_EP_DMA_METADATA_REG_LAYOUT_DATA, val);
+ if (layout_data == EDMA_MF_EDMA_LEGACY)
+ return -EOPNOTSUPP;
+ if (layout_data != EDMA_MF_EDMA_UNROLL &&
+ layout_data != EDMA_MF_HDMA_COMPAT &&
+ layout_data != EDMA_MF_HDMA_NATIVE)
+ return -EINVAL;
+
+ val = dw_edma_pcie_ep_dma_readl(view, PCI_EP_DMA_METADATA_CTRL);
+ reg_bar = FIELD_GET(PCI_EP_DMA_METADATA_CTRL_REG_BAR, val);
+ if (!dw_edma_pcie_valid_bar(reg_bar))
+ return -EINVAL;
+
+ wr_ch_cnt = FIELD_GET(PCI_EP_DMA_METADATA_CTRL_WR_CH_COUNT, val);
+ rd_ch_cnt = FIELD_GET(PCI_EP_DMA_METADATA_CTRL_RD_CH_COUNT, val);
+ if (!wr_ch_cnt && !rd_ch_cnt)
+ return -EINVAL;
+ if (wr_ch_cnt > EDMA_MAX_WR_CH || rd_ch_cnt > EDMA_MAX_RD_CH)
+ return -EINVAL;
+
+ entry_size = FIELD_GET(PCI_EP_DMA_METADATA_CTRL_CH_ENTRY_SIZE, val);
+ if (entry_size < PCI_EP_DMA_METADATA_CH_ENTRY_SIZE ||
+ entry_size % sizeof(u32))
+ return -EINVAL;
+
+ if (check_mul_overflow((size_t)(wr_ch_cnt + rd_ch_cnt),
+ (size_t)entry_size, &table_size) ||
+ check_add_overflow((size_t)PCI_EP_DMA_METADATA_HDR_LEN,
+ table_size, &table_end) ||
+ table_end > len)
+ return -EINVAL;
+
+ if (metadata_ctrl)
+ *metadata_ctrl = val;
+ if (reg_layout_data)
+ *reg_layout_data = layout_data;
+
+ return 0;
+}
+
+static int
+dw_edma_pcie_parse_ep_dma_data(struct dw_edma_pcie_ep_dma_view *view,
+ struct dw_edma_pcie_data *pdata)
+{
+ u32 ctrl, reg_sz;
+ u8 reg_layout_data;
+ u64 reg_off;
+ u16 wr_table, rd_table, entry_size;
+ u16 wr_ch_cnt, rd_ch_cnt;
+ int ret;
+
+ ret = dw_edma_pcie_validate_ep_dma_metadata(view, &ctrl,
+ ®_layout_data);
+ if (ret)
+ return ret;
+
+ pci_dbg(view->pdev, "Detected PCI endpoint DMA BAR metadata\n");
+
+ pdata->mf = reg_layout_data;
+ pdata->rg.bar = FIELD_GET(PCI_EP_DMA_METADATA_CTRL_REG_BAR, ctrl);
+
+ wr_ch_cnt = FIELD_GET(PCI_EP_DMA_METADATA_CTRL_WR_CH_COUNT, ctrl);
+ rd_ch_cnt = FIELD_GET(PCI_EP_DMA_METADATA_CTRL_RD_CH_COUNT, ctrl);
+ pdata->wr_ch_cnt = min_t(u16, pdata->wr_ch_cnt, wr_ch_cnt);
+ pdata->rd_ch_cnt = min_t(u16, pdata->rd_ch_cnt, rd_ch_cnt);
+ pdata->irqs = pdata->wr_ch_cnt + pdata->rd_ch_cnt;
+ reg_off = dw_edma_pcie_ep_dma_read64(view,
+ PCI_EP_DMA_METADATA_REG_OFF_LO,
+ PCI_EP_DMA_METADATA_REG_OFF_HI);
+ reg_sz = dw_edma_pcie_ep_dma_readl(view, PCI_EP_DMA_METADATA_REG_SIZE);
+ if (reg_off > type_max(pdata->rg.off) ||
+ !dw_edma_pcie_valid_bar_range(view->pdev, pdata->rg.bar,
+ reg_off, reg_sz))
+ return -EINVAL;
+ pdata->rg.off = reg_off;
+ pdata->rg.sz = reg_sz;
+
+ entry_size = FIELD_GET(PCI_EP_DMA_METADATA_CTRL_CH_ENTRY_SIZE, ctrl);
+ wr_table = PCI_EP_DMA_METADATA_HDR_LEN;
+ rd_table = PCI_EP_DMA_METADATA_HDR_LEN + wr_ch_cnt * entry_size;
+
+ ret = dw_edma_pcie_parse_ep_dma_ch_table(view, pdata, wr_table,
+ entry_size, pdata->wr_ch_cnt,
+ true);
+ if (ret)
+ return ret;
+
+ return dw_edma_pcie_parse_ep_dma_ch_table(view, pdata, rd_table,
+ entry_size,
+ pdata->rd_ch_cnt, false);
+}
+
+static int
+dw_edma_pcie_parse_ep_dma_caps(struct pci_dev *pdev,
+ struct dw_edma_pcie_data *pdata)
+{
+ struct dw_edma_pcie_ep_dma_view metadata_view;
+ void __iomem *base;
+ resource_size_t bar_len;
+ enum pci_barno bar;
+ u32 ctrl;
+ int ret;
+
+ for (bar = BAR_0; bar < PCI_STD_NUM_BARS; bar++) {
+ if (!dw_edma_pcie_ep_dma_bar_scannable(pdev, bar))
+ continue;
+
+ bar_len = pci_resource_len(pdev, bar);
+ base = pci_iomap_range(pdev, bar, 0, 0);
+ if (!base)
+ continue;
+
+ metadata_view = (struct dw_edma_pcie_ep_dma_view) {
+ .pdev = pdev,
+ .base = base,
+ .limit = bar_len,
+ };
+ ret = dw_edma_pcie_validate_ep_dma_metadata(&metadata_view,
+ NULL, NULL);
+ if (ret == -ENODEV) {
+ pci_iounmap(metadata_view.pdev, base);
+ continue;
+ }
+ if (ret) {
+ pci_iounmap(metadata_view.pdev, base);
+ return ret;
+ }
+
+ ctrl = dw_edma_pcie_ep_dma_readl(&metadata_view,
+ PCI_EP_DMA_METADATA_CTRL);
+ ctrl |= PCI_EP_DMA_METADATA_CTRL_HOST_REQ;
+ dw_edma_pcie_ep_dma_writel(&metadata_view,
+ PCI_EP_DMA_METADATA_CTRL, ctrl);
+
+ ret = dw_edma_pcie_ep_dma_wait_ready(&metadata_view);
+ if (ret) {
+ pci_iounmap(metadata_view.pdev, base);
+ return ret;
+ }
+
+ ret = dw_edma_pcie_parse_ep_dma_data(&metadata_view, pdata);
+ pci_iounmap(metadata_view.pdev, base);
+
+ return ret;
+ }
+
+ return -ENODEV;
+}
+
static int
dw_edma_pcie_parse_synopsys_caps(struct pci_dev *pdev,
struct dw_edma_pcie_data *pdata)
@@ -367,6 +726,14 @@ dw_edma_pcie_parse_xilinx_caps(struct pci_dev *pdev,
return 0;
}
+static const struct dw_edma_pcie_match_data ep_dma_match_data = {
+ .data = &ep_dma_data,
+ .plat_ops = &dw_edma_pcie_raw_addr_plat_ops,
+ .parse_caps = dw_edma_pcie_parse_ep_dma_caps,
+ .flags = DW_EDMA_PCIE_F_REG_OFFSET,
+ .chip_flags = DW_EDMA_CHIP_PARTIAL,
+};
+
static u64 dw_edma_get_phys_addr(struct pci_dev *pdev,
const struct dw_edma_pcie_match_data *match,
struct dw_edma_pcie_data *pdata,
@@ -400,8 +767,17 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
int err, nr_irqs;
int i, mask;
- if (!match)
- return -ENODEV;
+ if (!match) {
+ /*
+ * The endpoint DMA metadata path has no static PCI ID yet.
+ * Accept it only for an explicit driver_override bind, not for
+ * arbitrary dynamic IDs without driver data.
+ */
+ if (!device_has_driver_override(&pdev->dev))
+ return -ENODEV;
+
+ match = &ep_dma_match_data;
+ }
pdata = match->data;
if (!pdata)
--
2.51.0
^ permalink raw reply related
* [PATCH v3 0/3] PCI: endpoint: Add PCI DMA endpoint function (part 3/3)
From: Koichiro Den @ 2026-06-20 17:08 UTC (permalink / raw)
To: Manivannan Sadhasivam, Krzysztof Wilczyński,
Kishon Vijay Abraham I, Bjorn Helgaas, Jonathan Corbet,
Shuah Khan, Vinod Koul, Frank Li, Arnd Bergmann, Damien Le Moal,
Niklas Cassel
Cc: Marek Vasut, Yoshihiro Shimoda, linux-pci, linux-doc,
linux-kernel, dmaengine
Hi,
This is v3, part 3 of three series for PCI endpoint DMA.
The three series are:
* part 1: dmaengine: dw-edma: Prepare for PCI EP DMA
* part 2: PCI: endpoint: Expose endpoint DMA resources
* part 3: PCI: endpoint: Add PCI DMA endpoint function
This series adds the host-side metadata parser, the pci-epf-dma endpoint
function driver, and documentation.
The endpoint function exposes selected endpoint-integrated DMA channels as
a separate PCI DMA controller function. The host-side dw-edma-pcie driver
discovers the BAR metadata, requests the final layout, and registers the
exposed channels with DMAengine. Host clients then submit transfers through
the regular DMAengine API. The endpoint function keeps the metadata BAR
stable and uses a separate DMA window BAR for resources that need dynamic
subrange mappings.
No fixed PCI ID is assigned by this series. Users provide the PCI
vendor/device ID through configfs and bind dw-edma-pcie explicitly, for
example with driver_override.
Dependencies
============
This series is based on linux-next next-20260619 and depends on parts 1
and 2:
[PATCH v3 00/13] dmaengine: dw-edma: Prepare for PCI EP DMA (part 1/3)
https://lore.kernel.org/dmaengine/20260620170040.3756043-1-den@valinux.co.jp/
[PATCH v3 0/5] PCI: endpoint: Expose endpoint DMA resources (part 2/3)
https://lore.kernel.org/linux-pci/20260620170438.3756593-1-den@valinux.co.jp/
Open question for the full series
=================================
One remaining design question is how to support endpoint controllers that
can expose only one PF. One option is to keep pci-epf-dma as a separate
function and require multi-function endpoint support. Another is to fold
the DMA functionality into vNTB for such platforms, similar to the earlier,
likely superseded, separate series:
[PATCH 00/15] PCI: endpoint: Remote DMA support via vNTB
https://lore.kernel.org/linux-pci/20260312165005.1148676-1-den@valinux.co.jp/
My intention is for the first real consumer to be an NTB netdev/transport
over vNTB, using this DMA path to accelerate data transfers. Embedding DMA
support in vNTB would make that acceleration available even on endpoint
controllers that do not support multiple functions. However, it would also
make the vNTB code significantly more complex. The separate PCI DMA EPF
model in this series keeps the design cleaner and more modular.
Note
====
This series touches both dmaengine and PCI endpoint code. I kept the
dw-edma-pcie metadata parser together with the endpoint function so the
metadata producer and consumer can be reviewed in one place.
If the general direction looks acceptable, the dw-edma-pcie patch may need
a dmaengine Ack if this series is routed through the PCI endpoint tree.
Tested on
=========
The RC-to-EP data path was tested with a small out-of-tree DMAengine
client. The host submits a DMA_MEM_TO_DEV transfer through dw-edma-pcie,
which uses a DesignWare eDMA read channel to copy host memory into
endpoint memory.
Tested with these endpoint/root-complex pairs:
* R-Car S4 EP + R-Car S4 RC:
eDMA unroll; DMA register window mapped through a BAR subrange
* RK3588 EP + CD8180 RC:
eDMA unroll; DMA register window fixed in BAR space
* SpacemiT K3 EP + CD8180 RC:
HDMA native linked-list; DMA register window fixed in BAR space
Note: The SpacemiT K3 test used the vendor Ubuntu kernel
(6.18.3-5-spacemit-generic), which includes pcie-spacemit-ep.c, with the
required prerequisite series backported.
---
Changelog
=========
Changes in v3:
- Select endpoint DMA match data before copying DMA data and require
driver_override for the generic endpoint DMA fallback. (Sashiko)
- Accept HDMA native linked-list endpoint DMA metadata.
- Consume logical DMA channels separately from descriptor memory resources.
(Sashiko)
- Delegate channels through the EPC DMA channel delegation API instead of
v2's EPC-provided DMAengine filter callbacks.
- Allow HDMA native linked-list channels to be delegated at channel
granularity.
- Preserve HOST_REQ across link-down and retry DMA window submaps on the
next link-up.
- Drop trailing colons from documentation subsection headings. (Randy)
- Document HDMA native linked-list mode support and the current non-LL
limitation.
Changes in v2:
- Follow the part 1/3 and part 2/3 v2 channel-claim model: pci-epf-dma
now claims delegated channels through DMAengine filter information from
EPC auxiliary resources.
- Select raw-address dw-edma-pcie platform ops from the endpoint DMA
match entry instead of using a match flag.
v2: https://lore.kernel.org/linux-pci/20260525063456.3317509-1-den@valinux.co.jp/
v1: https://lore.kernel.org/linux-pci/20260521063638.2843021-1-den@valinux.co.jp/
Best regards,
Koichiro
Koichiro Den (3):
dmaengine: dw-edma-pcie: Discover endpoint DMA metadata
PCI: endpoint: Add DMA endpoint function
Documentation: PCI: Add PCI DMA endpoint function documentation
Documentation/PCI/endpoint/index.rst | 2 +
.../PCI/endpoint/pci-dma-function.rst | 188 +++
Documentation/PCI/endpoint/pci-dma-howto.rst | 201 +++
drivers/dma/dw-edma/dw-edma-pcie.c | 380 ++++-
drivers/pci/endpoint/functions/Kconfig | 14 +
drivers/pci/endpoint/functions/Makefile | 1 +
drivers/pci/endpoint/functions/pci-epf-dma.c | 1420 +++++++++++++++++
7 files changed, 2204 insertions(+), 2 deletions(-)
create mode 100644 Documentation/PCI/endpoint/pci-dma-function.rst
create mode 100644 Documentation/PCI/endpoint/pci-dma-howto.rst
create mode 100644 drivers/pci/endpoint/functions/pci-epf-dma.c
--
2.51.0
^ permalink raw reply
* [PATCH v3 2/3] PCI: endpoint: Add DMA endpoint function
From: Koichiro Den @ 2026-06-20 17:08 UTC (permalink / raw)
To: Manivannan Sadhasivam, Krzysztof Wilczyński,
Kishon Vijay Abraham I, Bjorn Helgaas, Jonathan Corbet,
Shuah Khan, Vinod Koul, Frank Li, Arnd Bergmann, Damien Le Moal,
Niklas Cassel
Cc: Marek Vasut, Yoshihiro Shimoda, linux-pci, linux-doc,
linux-kernel, dmaengine
In-Reply-To: <20260620170844.3757241-1-den@valinux.co.jp>
Add pci-epf-dma, an endpoint function that exposes selected
endpoint-integrated DMA channels as a separate PCI DMA controller
function.
The function consumes EPC auxiliary DMA channel and descriptor memory
resources, delegates channels through the EPC DMA channel delegation API,
publishes a stable metadata BAR for host discovery, and uses a DMA
window BAR for DMA resources that are not already host-visible. For
DesignWare eDMA unroll and HDMA compatible layouts, channel delegation
is constrained to whole directions. HDMA native linked-list mode uses
per-channel registers and can delegate a dense channel prefix without
taking the whole direction.
After the host-side driver finds the metadata and requests the final
layout, the endpoint function programs DMA window BAR submaps and marks
the metadata ready. If the link drops after the host request is set,
clear only the ready bit and retry submap programming on the next
link-up without requiring the host driver to probe again.
If setup fails before the metadata is marked ready, release any delegated
channels without asking the EPC backend to quiesce them. Once the ready
bit has been set, teardown requests quiesce because the host may have
programmed the exposed DMA windows.
The endpoint function does not bake in a vendor/device ID. As with other
generic endpoint functions, users provide the PCI IDs through the common
EPF configfs header attributes.
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes in v3:
- Allow HDMA native linked-list channels to be delegated at channel
granularity.
- Consume logical DMA channels separately from descriptor memory
resources (Sashiko).
- Delegate channels through the EPC DMA channel delegation API instead
of v2's EPC-provided DMAengine filter callbacks.
- Preserve HOST_REQ across link-down and retry DMA window submaps on
link-up.
- Release delegated channels without backend quiesce when unwinding
setup before metadata READY.
drivers/pci/endpoint/functions/Kconfig | 14 +
drivers/pci/endpoint/functions/Makefile | 1 +
drivers/pci/endpoint/functions/pci-epf-dma.c | 1420 ++++++++++++++++++
3 files changed, 1435 insertions(+)
create mode 100644 drivers/pci/endpoint/functions/pci-epf-dma.c
diff --git a/drivers/pci/endpoint/functions/Kconfig b/drivers/pci/endpoint/functions/Kconfig
index bb5a23994288..078ac19dc772 100644
--- a/drivers/pci/endpoint/functions/Kconfig
+++ b/drivers/pci/endpoint/functions/Kconfig
@@ -39,6 +39,20 @@ config PCI_EPF_VNTB
If in doubt, say "N" to disable Endpoint NTB driver.
+config PCI_EPF_DMA
+ tristate "PCI Endpoint DMA driver"
+ depends on PCI_ENDPOINT
+ select CONFIGFS_FS
+ select DMA_ENGINE
+ help
+ Select this configuration option to expose an endpoint-integrated
+ DMA controller as a PCI endpoint function. The function advertises
+ the DMA controller layout to the host using BAR-resident metadata
+ and maps resources that are not already host-visible into the
+ DMA window BAR.
+
+ If in doubt, say "N" to disable Endpoint DMA driver.
+
config PCI_EPF_MHI
tristate "PCI Endpoint driver for MHI bus"
depends on PCI_ENDPOINT && MHI_BUS_EP
diff --git a/drivers/pci/endpoint/functions/Makefile b/drivers/pci/endpoint/functions/Makefile
index 696473fce50e..de92f6897b8f 100644
--- a/drivers/pci/endpoint/functions/Makefile
+++ b/drivers/pci/endpoint/functions/Makefile
@@ -6,4 +6,5 @@
obj-$(CONFIG_PCI_EPF_TEST) += pci-epf-test.o
obj-$(CONFIG_PCI_EPF_NTB) += pci-epf-ntb.o
obj-$(CONFIG_PCI_EPF_VNTB) += pci-epf-vntb.o
+obj-$(CONFIG_PCI_EPF_DMA) += pci-epf-dma.o
obj-$(CONFIG_PCI_EPF_MHI) += pci-epf-mhi.o
diff --git a/drivers/pci/endpoint/functions/pci-epf-dma.c b/drivers/pci/endpoint/functions/pci-epf-dma.c
new file mode 100644
index 000000000000..cd8c2ae39c07
--- /dev/null
+++ b/drivers/pci/endpoint/functions/pci-epf-dma.c
@@ -0,0 +1,1420 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * PCI endpoint function that exposes an endpoint-integrated DMA controller
+ * to the PCI host.
+ *
+ * The host-side dw-edma-pcie driver consumes the BAR metadata published
+ * by this function.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/configfs.h>
+#include <linux/dma/edma.h>
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include <linux/overflow.h>
+#include <linux/pci-ep-dma.h>
+#include <linux/pci-epc.h>
+#include <linux/pci-epf.h>
+#include <linux/pci_regs.h>
+#include <linux/slab.h>
+#include <linux/workqueue.h>
+
+/* HOST_REQ is set by the host driver, so poll it at a low idle rate. */
+#define PCI_EPF_DMA_HOST_REQ_POLL_MS 500
+
+struct pci_epf_dma_bar_map {
+ const struct pci_epc_aux_resource *res;
+ enum pci_barno bar;
+ u64 res_offset_in_bar;
+ u64 submap_offset_in_bar;
+ dma_addr_t phys_addr;
+ size_t map_size;
+ bool needs_submap;
+};
+
+struct pci_epf_dma {
+ struct pci_epf *epf;
+ struct config_group group;
+ struct delayed_work map_work;
+
+ enum pci_barno metadata_bar;
+ enum pci_barno dma_window_bar;
+ u16 wr_chans;
+ u16 rd_chans;
+ u8 reg_layout;
+ u8 reg_layout_data;
+
+ /* Backing storage for ctrl, channel and descriptor resource pointers. */
+ struct pci_epc_aux_resource *resources;
+ unsigned int num_resources;
+ const struct pci_epc_aux_resource *ctrl;
+ const struct pci_epc_aux_resource *ep_to_rc_aux_chan[EDMA_MAX_WR_CH];
+ const struct pci_epc_aux_resource *rc_to_ep_aux_chan[EDMA_MAX_RD_CH];
+ const struct pci_epc_aux_resource *ep_to_rc_desc[EDMA_MAX_WR_CH];
+ const struct pci_epc_aux_resource *rc_to_ep_desc[EDMA_MAX_RD_CH];
+
+ /* Local EPC reservations for channels delegated to the host. */
+ struct pci_epc_dma_chan *ep_to_rc_chan[EDMA_MAX_WR_CH];
+ struct pci_epc_dma_chan *rc_to_ep_chan[EDMA_MAX_RD_CH];
+
+ void *metadata_addr;
+ void *dma_window_addr;
+ size_t msix_table_offset;
+ struct pci_epf_dma_bar_map *bar_maps;
+ unsigned int num_bar_maps;
+ struct pci_epf_bar_submap *submaps;
+ unsigned int num_submaps;
+
+ /* Cleared when a later event should retry programming the submaps. */
+ bool submaps_programmed;
+ bool channels_exposed;
+};
+
+#define to_epf_dma(epf_group) container_of((epf_group), struct pci_epf_dma, group)
+
+static struct pci_epf_header pci_epf_dma_header = {
+ .vendorid = PCI_ANY_ID,
+ .deviceid = PCI_ANY_ID,
+ .baseclass_code = PCI_BASE_CLASS_SYSTEM,
+ .subclass_code = PCI_CLASS_SYSTEM_DMA & 0xff,
+ .interrupt_pin = PCI_INTERRUPT_INTA,
+};
+
+static void pci_epf_dma_release_channels(struct pci_epf_dma *epf_dma)
+{
+ bool quiesce = epf_dma->channels_exposed;
+ unsigned int i;
+
+ for (i = 0; i < ARRAY_SIZE(epf_dma->ep_to_rc_chan); i++) {
+ if (!epf_dma->ep_to_rc_chan[i])
+ continue;
+
+ pci_epc_reclaim_dma_chan(epf_dma->ep_to_rc_chan[i], quiesce);
+ epf_dma->ep_to_rc_chan[i] = NULL;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(epf_dma->rc_to_ep_chan); i++) {
+ if (!epf_dma->rc_to_ep_chan[i])
+ continue;
+
+ pci_epc_reclaim_dma_chan(epf_dma->rc_to_ep_chan[i], quiesce);
+ epf_dma->rc_to_ep_chan[i] = NULL;
+ }
+
+ epf_dma->channels_exposed = false;
+}
+
+static int pci_epf_dma_claim_channel(struct pci_epf_dma *epf_dma,
+ const struct pci_epc_aux_resource *res,
+ struct pci_epc_dma_chan **chan)
+{
+ struct pci_epf *epf = epf_dma->epf;
+ struct device *dev = &epf_dma->epf->dev;
+ int ret;
+
+ ret = pci_epc_delegate_dma_chan(epf->epc, epf->func_no, epf->vfunc_no,
+ res->u.dma_chan.dir,
+ res->u.dma_chan.hw_ch, chan);
+ if (ret) {
+ dev_err(dev, "DMA channel %u cannot be delegated\n",
+ res->u.dma_chan.hw_ch);
+ return ret;
+ }
+
+ return 0;
+}
+
+static int
+pci_epf_dma_validate_dw_edma_ctrl(struct pci_epf_dma *epf_dma,
+ const struct pci_epc_aux_resource *ctrl)
+{
+ struct device *dev = &epf_dma->epf->dev;
+ enum dw_edma_map_format map = ctrl->u.dma_ctrl.reg_layout_data;
+ u16 total_wr_chans = ctrl->u.dma_ctrl.ep_to_rc_ch_cnt;
+ u16 total_rd_chans = ctrl->u.dma_ctrl.rc_to_ep_ch_cnt;
+
+ switch (map) {
+ case EDMA_MF_EDMA_LEGACY:
+ dev_err(dev, "legacy DesignWare eDMA layout cannot be delegated\n");
+ return -EOPNOTSUPP;
+ case EDMA_MF_EDMA_UNROLL:
+ case EDMA_MF_HDMA_COMPAT:
+ if ((epf_dma->wr_chans && epf_dma->wr_chans != total_wr_chans) ||
+ (epf_dma->rd_chans && epf_dma->rd_chans != total_rd_chans)) {
+ dev_err(dev, "DesignWare eDMA v0 delegation must cover the whole direction\n");
+ return -EOPNOTSUPP;
+ }
+ return 0;
+ case EDMA_MF_HDMA_NATIVE:
+ return 0;
+ default:
+ return -EINVAL;
+ }
+}
+
+static bool pci_epf_dma_bar_usable(const struct pci_epc_features *epc_features,
+ enum pci_barno bar)
+{
+ if (bar < BAR_0 || bar >= PCI_STD_NUM_BARS)
+ return false;
+
+ return epc_features->bar[bar].type != BAR_RESERVED &&
+ epc_features->bar[bar].type != BAR_DISABLED;
+}
+
+static bool pci_epf_dma_bar_has_fixed_resource(struct pci_epf_dma *epf_dma,
+ enum pci_barno bar)
+{
+ unsigned int i;
+
+ for (i = 0; i < epf_dma->num_resources; i++) {
+ if (epf_dma->resources[i].bar == bar)
+ return true;
+ }
+
+ return false;
+}
+
+static enum pci_barno
+pci_epf_dma_first_usable_bar(struct pci_epf_dma *epf_dma,
+ const struct pci_epc_features *epc_features,
+ enum pci_barno exclude)
+{
+ enum pci_barno bar;
+
+ for (bar = BAR_0; bar < PCI_STD_NUM_BARS; bar++) {
+ bar = pci_epc_get_next_free_bar(epc_features, bar);
+ if (bar == NO_BAR)
+ return NO_BAR;
+ if (bar != exclude &&
+ !pci_epf_dma_bar_has_fixed_resource(epf_dma, bar))
+ return bar;
+ }
+
+ return NO_BAR;
+}
+
+static size_t pci_epf_dma_align_size(size_t size, size_t align)
+{
+ if (!align)
+ return size;
+
+ return ALIGN(size, align);
+}
+
+static int pci_epf_dma_reuse_submap(struct pci_epf_dma *epf_dma,
+ unsigned int map_count,
+ dma_addr_t phys_addr, size_t map_size,
+ size_t offset, size_t *next_offset_in_bar,
+ u64 *res_offset_in_bar)
+{
+ struct pci_epf_dma_bar_map *map;
+ u64 delta;
+ size_t merged_size, next;
+ u64 res_map_end, submap_bar_end, submap_phys_end;
+ unsigned int i;
+
+ if (check_add_overflow(phys_addr, map_size, &res_map_end))
+ return -EOVERFLOW;
+
+ for (i = 0; i < map_count; i++) {
+ map = &epf_dma->bar_maps[i];
+ if (!map->needs_submap || map->bar != epf_dma->dma_window_bar)
+ continue;
+
+ if (check_add_overflow(map->phys_addr, map->map_size,
+ &submap_phys_end) ||
+ check_add_overflow(map->submap_offset_in_bar,
+ map->map_size, &submap_bar_end))
+ return -EOVERFLOW;
+
+ /*
+ * Reuse a submap that already covers this aligned resource
+ * window.
+ */
+ if (phys_addr >= map->phys_addr &&
+ res_map_end <= submap_phys_end) {
+ if (check_add_overflow(phys_addr - map->phys_addr,
+ offset, &delta) ||
+ check_add_overflow(map->submap_offset_in_bar,
+ delta, res_offset_in_bar))
+ return -EOVERFLOW;
+ return 1;
+ }
+
+ /*
+ * Extend only the BAR-tail submap when the physical ranges are
+ * contiguous.
+ */
+ if (submap_phys_end == phys_addr &&
+ submap_bar_end == *next_offset_in_bar) {
+ if (check_add_overflow(map->map_size, map_size,
+ &merged_size) ||
+ check_add_overflow(*next_offset_in_bar, map_size,
+ &next) ||
+ check_add_overflow(*next_offset_in_bar, offset,
+ res_offset_in_bar))
+ return -EOVERFLOW;
+
+ map->map_size = merged_size;
+ *next_offset_in_bar = next;
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+static int pci_epf_dma_add_map(struct pci_epf_dma *epf_dma,
+ const struct pci_epc_aux_resource *res,
+ size_t align, size_t *next_offset_in_bar,
+ unsigned int *map_idx)
+{
+ dma_addr_t phys_addr;
+ size_t map_size, offset = 0, next;
+ u64 res_offset_in_bar;
+ unsigned int i;
+ int ret;
+
+ if (!res || !res->size)
+ return -EINVAL;
+
+ for (i = 0; i < *map_idx; i++) {
+ if (epf_dma->bar_maps[i].res == res)
+ return 0;
+ }
+
+ if (res->bar != NO_BAR) {
+ if (res->bar < BAR_0 || res->bar >= PCI_STD_NUM_BARS)
+ return -EINVAL;
+ if (res->bar == epf_dma->metadata_bar ||
+ res->bar == epf_dma->dma_window_bar)
+ return -EINVAL;
+
+ epf_dma->bar_maps[*map_idx] = (struct pci_epf_dma_bar_map) {
+ .res = res,
+ .bar = res->bar,
+ .res_offset_in_bar = res->bar_offset,
+ .map_size = res->size,
+ };
+ (*map_idx)++;
+
+ return 0;
+ }
+
+ if (epf_dma->dma_window_bar == NO_BAR)
+ return -EOPNOTSUPP;
+
+ phys_addr = res->phys_addr;
+ /* Map the aligned window that contains this resource. */
+ if (align) {
+ phys_addr = ALIGN_DOWN(res->phys_addr, align);
+ offset = res->phys_addr - phys_addr;
+ }
+
+ if (check_add_overflow(res->size, offset, &map_size))
+ return -EOVERFLOW;
+ map_size = pci_epf_dma_align_size(map_size, align);
+
+ ret = pci_epf_dma_reuse_submap(epf_dma, *map_idx, phys_addr, map_size,
+ offset, next_offset_in_bar,
+ &res_offset_in_bar);
+ if (ret < 0)
+ return ret;
+ if (ret) {
+ epf_dma->bar_maps[*map_idx] = (struct pci_epf_dma_bar_map) {
+ .res = res,
+ .bar = epf_dma->dma_window_bar,
+ .res_offset_in_bar = res_offset_in_bar,
+ .phys_addr = res->phys_addr,
+ .map_size = res->size,
+ };
+
+ (*map_idx)++;
+
+ return 0;
+ }
+
+ if (check_add_overflow(*next_offset_in_bar, map_size, &next))
+ return -EOVERFLOW;
+ if (check_add_overflow(*next_offset_in_bar, offset, &res_offset_in_bar))
+ return -EOVERFLOW;
+
+ epf_dma->bar_maps[*map_idx] = (struct pci_epf_dma_bar_map) {
+ .res = res,
+ .bar = epf_dma->dma_window_bar,
+ .res_offset_in_bar = res_offset_in_bar,
+ .submap_offset_in_bar = *next_offset_in_bar,
+ .phys_addr = phys_addr,
+ .map_size = map_size,
+ .needs_submap = true,
+ };
+
+ *next_offset_in_bar = next;
+ (*map_idx)++;
+
+ return 0;
+}
+
+static const struct pci_epf_dma_bar_map *
+pci_epf_dma_find_map(struct pci_epf_dma *epf_dma,
+ const struct pci_epc_aux_resource *res)
+{
+ unsigned int i;
+
+ for (i = 0; i < epf_dma->num_bar_maps; i++) {
+ if (epf_dma->bar_maps[i].res == res)
+ return &epf_dma->bar_maps[i];
+ }
+
+ return NULL;
+}
+
+static bool pci_epf_dma_needs_dma_window(struct pci_epf_dma *epf_dma)
+{
+ unsigned int i;
+
+ if (epf_dma->ctrl && epf_dma->ctrl->bar == NO_BAR)
+ return true;
+
+ for (i = 0; i < epf_dma->wr_chans; i++) {
+ if (epf_dma->ep_to_rc_desc[i] &&
+ epf_dma->ep_to_rc_desc[i]->bar == NO_BAR)
+ return true;
+ }
+
+ for (i = 0; i < epf_dma->rd_chans; i++) {
+ if (epf_dma->rc_to_ep_desc[i] &&
+ epf_dma->rc_to_ep_desc[i]->bar == NO_BAR)
+ return true;
+ }
+
+ return false;
+}
+
+static const struct pci_epc_aux_resource *
+pci_epf_dma_find_desc_mem(const struct pci_epc_aux_resource *res, int count,
+ u16 id)
+{
+ int i;
+
+ for (i = 0; i < count; i++) {
+ if (res[i].type == PCI_EPC_AUX_DMA_DESC_MEM &&
+ res[i].u.dma_desc.id == id)
+ return &res[i];
+ }
+
+ return NULL;
+}
+
+static int pci_epf_dma_collect_resources(struct pci_epf_dma *epf_dma)
+{
+ const struct pci_epc_aux_resource *ep_to_rc_aux_chan[EDMA_MAX_WR_CH] = {};
+ const struct pci_epc_aux_resource *rc_to_ep_aux_chan[EDMA_MAX_RD_CH] = {};
+ const struct pci_epc_aux_resource *ep_to_rc_desc[EDMA_MAX_WR_CH] = {};
+ const struct pci_epc_aux_resource *rc_to_ep_desc[EDMA_MAX_RD_CH] = {};
+ const struct pci_epc_aux_resource *ctrl = NULL;
+ struct pci_epf *epf = epf_dma->epf;
+ struct pci_epc *epc = epf->epc;
+ struct device *dev = &epf->dev;
+ int count, i, ret;
+
+ count = pci_epc_get_aux_resources_count(epc, epf->func_no,
+ epf->vfunc_no);
+ if (count <= 0)
+ return count ?: -ENODEV;
+
+ struct pci_epc_aux_resource *res __free(kfree) =
+ kzalloc_objs(*res, count);
+ if (!res)
+ return -ENOMEM;
+
+ ret = pci_epc_get_aux_resources(epc, epf->func_no, epf->vfunc_no,
+ res, count);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < count; i++) {
+ switch (res[i].type) {
+ case PCI_EPC_AUX_DMA_CTRL_MMIO:
+ if (ctrl)
+ return -EINVAL;
+ ctrl = &res[i];
+ break;
+ case PCI_EPC_AUX_DMA_CHAN: {
+ u16 hw_ch = res[i].u.dma_chan.hw_ch;
+
+ switch (res[i].u.dma_chan.dir) {
+ case PCI_EPC_AUX_DMA_EP_TO_RC:
+ if (hw_ch >= EDMA_MAX_WR_CH ||
+ ep_to_rc_aux_chan[hw_ch])
+ return -EINVAL;
+ ep_to_rc_aux_chan[hw_ch] = &res[i];
+ break;
+ case PCI_EPC_AUX_DMA_RC_TO_EP:
+ if (hw_ch >= EDMA_MAX_RD_CH ||
+ rc_to_ep_aux_chan[hw_ch])
+ return -EINVAL;
+ rc_to_ep_aux_chan[hw_ch] = &res[i];
+ break;
+ default:
+ return -EINVAL;
+ }
+ break;
+ }
+ case PCI_EPC_AUX_DMA_DESC_MEM:
+ if (pci_epf_dma_find_desc_mem(res, i,
+ res[i].u.dma_desc.id))
+ return -EINVAL;
+ break;
+ default:
+ continue;
+ }
+ }
+
+ if (!ctrl)
+ return -ENODEV;
+
+ if (!epf_dma->wr_chans && !epf_dma->rd_chans)
+ return -EINVAL;
+
+ if (epf_dma->wr_chans > ctrl->u.dma_ctrl.ep_to_rc_ch_cnt ||
+ epf_dma->rd_chans > ctrl->u.dma_ctrl.rc_to_ep_ch_cnt)
+ return -EINVAL;
+
+ switch (ctrl->u.dma_ctrl.reg_layout) {
+ case PCI_EPC_AUX_DMA_REG_LAYOUT_DW_EDMA:
+ ret = pci_epf_dma_validate_dw_edma_ctrl(epf_dma, ctrl);
+ if (ret)
+ return ret;
+ epf_dma->reg_layout = PCI_EP_DMA_METADATA_REG_LAYOUT_DW_EDMA;
+ epf_dma->reg_layout_data = ctrl->u.dma_ctrl.reg_layout_data;
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ for (i = 0; i < epf_dma->wr_chans; i++) {
+ if (!ep_to_rc_aux_chan[i]) {
+ dev_err(dev, "missing dense write DMA channel %d\n", i);
+ return -EINVAL;
+ }
+ }
+
+ for (i = 0; i < epf_dma->rd_chans; i++) {
+ if (!rc_to_ep_aux_chan[i]) {
+ dev_err(dev, "missing dense read DMA channel %d\n", i);
+ return -EINVAL;
+ }
+ }
+
+ for (i = 0; i < epf_dma->wr_chans; i++) {
+ u16 desc_mem_id = ep_to_rc_aux_chan[i]->u.dma_chan.desc_mem_id;
+
+ ep_to_rc_desc[i] = pci_epf_dma_find_desc_mem(res, count, desc_mem_id);
+ if (!ep_to_rc_desc[i]) {
+ dev_err(dev, "missing write DMA descriptor memory %d\n", i);
+ return -EINVAL;
+ }
+ }
+
+ for (i = 0; i < epf_dma->rd_chans; i++) {
+ u16 desc_mem_id = rc_to_ep_aux_chan[i]->u.dma_chan.desc_mem_id;
+
+ rc_to_ep_desc[i] = pci_epf_dma_find_desc_mem(res, count, desc_mem_id);
+ if (!rc_to_ep_desc[i]) {
+ dev_err(dev, "missing read DMA descriptor memory %d\n", i);
+ return -EINVAL;
+ }
+ }
+
+ for (i = 0; i < epf_dma->wr_chans; i++) {
+ ret = pci_epf_dma_claim_channel(epf_dma, ep_to_rc_aux_chan[i],
+ &epf_dma->ep_to_rc_chan[i]);
+ if (ret)
+ goto err_release_channels;
+ }
+
+ for (i = 0; i < epf_dma->rd_chans; i++) {
+ ret = pci_epf_dma_claim_channel(epf_dma, rc_to_ep_aux_chan[i],
+ &epf_dma->rc_to_ep_chan[i]);
+ if (ret)
+ goto err_release_channels;
+ }
+
+ epf_dma->resources = no_free_ptr(res);
+ epf_dma->num_resources = count;
+ epf_dma->ctrl = ctrl;
+ memcpy(epf_dma->ep_to_rc_aux_chan, ep_to_rc_aux_chan,
+ sizeof(ep_to_rc_aux_chan));
+ memcpy(epf_dma->rc_to_ep_aux_chan, rc_to_ep_aux_chan,
+ sizeof(rc_to_ep_aux_chan));
+ memcpy(epf_dma->ep_to_rc_desc, ep_to_rc_desc, sizeof(ep_to_rc_desc));
+ memcpy(epf_dma->rc_to_ep_desc, rc_to_ep_desc, sizeof(rc_to_ep_desc));
+
+ return 0;
+
+err_release_channels:
+ pci_epf_dma_release_channels(epf_dma);
+
+ return ret;
+}
+
+static void pci_epf_dma_metadata_write(__le32 *metadata, u16 metadata_off,
+ u32 val)
+{
+ metadata[metadata_off / sizeof(*metadata)] = cpu_to_le32(val);
+}
+
+static void pci_epf_dma_metadata_write64(__le32 *metadata, u16 metadata_off,
+ u64 val)
+{
+ pci_epf_dma_metadata_write(metadata, metadata_off, lower_32_bits(val));
+ pci_epf_dma_metadata_write(metadata, metadata_off + sizeof(u32),
+ upper_32_bits(val));
+}
+
+static int pci_epf_dma_build_ch_entry(const struct pci_epc_aux_resource *chan,
+ const struct pci_epf_dma_bar_map *map,
+ __le32 *metadata, u16 entry)
+{
+ const struct pci_epc_aux_resource *res = map->res;
+ u32 ctrl;
+
+ if (res->size > U32_MAX)
+ return -EOVERFLOW;
+
+ ctrl = FIELD_PREP(PCI_EP_DMA_METADATA_CH_CTRL_HW_CH,
+ chan->u.dma_chan.hw_ch) |
+ FIELD_PREP(PCI_EP_DMA_METADATA_CH_CTRL_DESC_BAR, map->bar);
+
+ pci_epf_dma_metadata_write(metadata, entry + PCI_EP_DMA_METADATA_CH_CTRL,
+ ctrl);
+ pci_epf_dma_metadata_write64(metadata,
+ entry + PCI_EP_DMA_METADATA_CH_DESC_OFF_LO,
+ map->res_offset_in_bar);
+ pci_epf_dma_metadata_write(metadata,
+ entry + PCI_EP_DMA_METADATA_CH_DESC_SIZE,
+ (u32)res->size);
+ pci_epf_dma_metadata_write64(metadata,
+ entry + PCI_EP_DMA_METADATA_CH_DESC_ADDR_LO,
+ res->phys_addr);
+
+ return 0;
+}
+
+static void pci_epf_dma_set_metadata_ready(struct pci_epf_dma *epf_dma,
+ bool ready)
+{
+ __le32 *metadata = epf_dma->metadata_addr;
+ __le32 *ctrl_ptr;
+ u32 ctrl;
+
+ if (!metadata)
+ return;
+
+ ctrl_ptr = &metadata[PCI_EP_DMA_METADATA_CTRL / sizeof(*metadata)];
+ ctrl = le32_to_cpu(READ_ONCE(*ctrl_ptr));
+ if (ready) {
+ dma_wmb();
+ ctrl |= PCI_EP_DMA_METADATA_CTRL_READY;
+ } else {
+ ctrl &= ~PCI_EP_DMA_METADATA_CTRL_READY;
+ }
+ WRITE_ONCE(*ctrl_ptr, cpu_to_le32(ctrl));
+ if (ready)
+ epf_dma->channels_exposed = true;
+}
+
+static bool pci_epf_dma_metadata_host_requested(struct pci_epf_dma *epf_dma)
+{
+ __le32 *metadata = epf_dma->metadata_addr;
+ u32 ctrl;
+
+ if (!metadata)
+ return false;
+
+ ctrl = le32_to_cpu(READ_ONCE(metadata[PCI_EP_DMA_METADATA_CTRL /
+ sizeof(*metadata)]));
+
+ return ctrl & PCI_EP_DMA_METADATA_CTRL_HOST_REQ;
+}
+
+static void pci_epf_dma_clear_metadata_status(struct pci_epf_dma *epf_dma)
+{
+ __le32 *metadata = epf_dma->metadata_addr;
+ __le32 *ctrl_ptr;
+ u32 ctrl;
+
+ if (!metadata)
+ return;
+
+ ctrl_ptr = &metadata[PCI_EP_DMA_METADATA_CTRL / sizeof(*metadata)];
+ ctrl = le32_to_cpu(READ_ONCE(*ctrl_ptr));
+ ctrl &= ~(PCI_EP_DMA_METADATA_CTRL_HOST_REQ |
+ PCI_EP_DMA_METADATA_CTRL_READY);
+ WRITE_ONCE(*ctrl_ptr, cpu_to_le32(ctrl));
+}
+
+static int pci_epf_dma_build_metadata(struct pci_epf_dma *epf_dma)
+{
+ const struct pci_epf_dma_bar_map *ctrl_map;
+ u16 entry_size = PCI_EP_DMA_METADATA_CH_ENTRY_SIZE;
+ u16 wr_table, rd_table, total_len;
+ __le32 *metadata = epf_dma->metadata_addr;
+ unsigned int i;
+ int ret;
+
+ if (!metadata)
+ return -EINVAL;
+
+ ctrl_map = pci_epf_dma_find_map(epf_dma, epf_dma->ctrl);
+ if (!ctrl_map)
+ return -EINVAL;
+ if (epf_dma->wr_chans > FIELD_MAX(PCI_EP_DMA_METADATA_CTRL_WR_CH_COUNT) ||
+ epf_dma->rd_chans > FIELD_MAX(PCI_EP_DMA_METADATA_CTRL_RD_CH_COUNT) ||
+ entry_size > FIELD_MAX(PCI_EP_DMA_METADATA_CTRL_CH_ENTRY_SIZE) ||
+ ctrl_map->res->size > U32_MAX)
+ return -EOVERFLOW;
+
+ wr_table = epf_dma->wr_chans ? PCI_EP_DMA_METADATA_HDR_LEN : 0;
+ rd_table = epf_dma->rd_chans ?
+ PCI_EP_DMA_METADATA_HDR_LEN + epf_dma->wr_chans * entry_size : 0;
+ total_len = PCI_EP_DMA_METADATA_HDR_LEN +
+ (epf_dma->wr_chans + epf_dma->rd_chans) * entry_size;
+
+ memset(metadata, 0, total_len);
+
+ pci_epf_dma_metadata_write(metadata, 0, PCI_EP_DMA_METADATA_MAGIC);
+ pci_epf_dma_metadata_write(metadata, PCI_EP_DMA_METADATA_HDR,
+ FIELD_PREP(PCI_EP_DMA_METADATA_HDR_REV,
+ PCI_EP_DMA_METADATA_REV) |
+ FIELD_PREP(PCI_EP_DMA_METADATA_HDR_LEN_FIELD,
+ total_len));
+ pci_epf_dma_metadata_write(metadata, PCI_EP_DMA_METADATA_CTRL,
+ FIELD_PREP(PCI_EP_DMA_METADATA_CTRL_REG_BAR,
+ ctrl_map->bar) |
+ FIELD_PREP(PCI_EP_DMA_METADATA_CTRL_WR_CH_COUNT,
+ epf_dma->wr_chans) |
+ FIELD_PREP(PCI_EP_DMA_METADATA_CTRL_RD_CH_COUNT,
+ epf_dma->rd_chans) |
+ FIELD_PREP(PCI_EP_DMA_METADATA_CTRL_CH_ENTRY_SIZE,
+ entry_size));
+ pci_epf_dma_metadata_write64(metadata,
+ PCI_EP_DMA_METADATA_REG_OFF_LO,
+ ctrl_map->res_offset_in_bar);
+ pci_epf_dma_metadata_write(metadata, PCI_EP_DMA_METADATA_REG_LAYOUT,
+ FIELD_PREP(PCI_EP_DMA_METADATA_REG_LAYOUT_ID,
+ epf_dma->reg_layout) |
+ FIELD_PREP(PCI_EP_DMA_METADATA_REG_LAYOUT_DATA,
+ epf_dma->reg_layout_data));
+ pci_epf_dma_metadata_write(metadata, PCI_EP_DMA_METADATA_REG_SIZE,
+ (u32)ctrl_map->res->size);
+
+ for (i = 0; i < epf_dma->wr_chans; i++) {
+ const struct pci_epf_dma_bar_map *map;
+
+ map = pci_epf_dma_find_map(epf_dma,
+ epf_dma->ep_to_rc_desc[i]);
+ if (!map)
+ return -EINVAL;
+ ret = pci_epf_dma_build_ch_entry(epf_dma->ep_to_rc_aux_chan[i],
+ map, metadata,
+ wr_table + i * entry_size);
+ if (ret)
+ return ret;
+ }
+
+ for (i = 0; i < epf_dma->rd_chans; i++) {
+ const struct pci_epf_dma_bar_map *map;
+
+ map = pci_epf_dma_find_map(epf_dma,
+ epf_dma->rc_to_ep_desc[i]);
+ if (!map)
+ return -EINVAL;
+ ret = pci_epf_dma_build_ch_entry(epf_dma->rc_to_ep_aux_chan[i],
+ map, metadata,
+ rd_table + i * entry_size);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static int pci_epf_dma_reserve_msix(struct pci_epf_dma *epf_dma,
+ const struct pci_epc_features *epc_features,
+ size_t *backing_size)
+{
+ struct pci_epf *epf = epf_dma->epf;
+ size_t msix_table_size, pba_size, next;
+ unsigned int nvec = epf->msix_interrupts;
+
+ epf_dma->msix_table_offset = 0;
+
+ if (!epc_features->msix_capable || !nvec)
+ return 0;
+
+ next = ALIGN(*backing_size, 8);
+ if (next > U32_MAX)
+ return -EOVERFLOW;
+ epf_dma->msix_table_offset = next;
+
+ if (check_mul_overflow(PCI_MSIX_ENTRY_SIZE, nvec, &msix_table_size))
+ return -EOVERFLOW;
+
+ pba_size = ALIGN(DIV_ROUND_UP(nvec, 8), 8);
+ if (check_add_overflow(next, msix_table_size, &next) ||
+ next > U32_MAX ||
+ check_add_overflow(next, pba_size, &next))
+ return -EOVERFLOW;
+
+ *backing_size = next;
+
+ return 0;
+}
+
+static int pci_epf_dma_build_layout(struct pci_epf_dma *epf_dma,
+ const struct pci_epc_features *epc_features)
+{
+ struct pci_epf *epf = epf_dma->epf;
+ struct device *dev = &epf->dev;
+ struct pci_epf_bar *bar;
+ unsigned int max_maps, map_idx = 0, sub_idx = 0;
+ size_t align = epc_features->align;
+ size_t metadata_size, metadata_backing_size, metadata_bar_size;
+ size_t mapped_size = 0, dma_window_bar_size;
+ int i, ret;
+
+ metadata_size = PCI_EP_DMA_METADATA_HDR_LEN;
+ metadata_size += (epf_dma->wr_chans + epf_dma->rd_chans) *
+ PCI_EP_DMA_METADATA_CH_ENTRY_SIZE;
+ metadata_backing_size = metadata_size;
+ ret = pci_epf_dma_reserve_msix(epf_dma, epc_features,
+ &metadata_backing_size);
+ if (ret)
+ return ret;
+ metadata_bar_size = pci_epf_dma_align_size(metadata_backing_size,
+ align);
+
+ epf_dma->metadata_addr = pci_epf_alloc_space(epf, metadata_bar_size,
+ epf_dma->metadata_bar,
+ epc_features,
+ PRIMARY_INTERFACE);
+ if (!epf_dma->metadata_addr) {
+ dev_err(dev, "failed to allocate BAR%d metadata space\n",
+ epf_dma->metadata_bar);
+ return -ENOMEM;
+ }
+ memset(epf_dma->metadata_addr, 0, epf->bar[epf_dma->metadata_bar].size);
+
+ /* One map for DMA controller registers, plus one per channel. */
+ max_maps = 1 + epf_dma->wr_chans + epf_dma->rd_chans;
+ epf_dma->bar_maps = kzalloc_objs(*epf_dma->bar_maps, max_maps);
+ if (!epf_dma->bar_maps)
+ return -ENOMEM;
+
+ ret = pci_epf_dma_add_map(epf_dma, epf_dma->ctrl, align,
+ &mapped_size, &map_idx);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < epf_dma->wr_chans; i++) {
+ ret = pci_epf_dma_add_map(epf_dma,
+ epf_dma->ep_to_rc_desc[i], align,
+ &mapped_size, &map_idx);
+ if (ret)
+ return ret;
+ }
+
+ for (i = 0; i < epf_dma->rd_chans; i++) {
+ ret = pci_epf_dma_add_map(epf_dma,
+ epf_dma->rc_to_ep_desc[i], align,
+ &mapped_size, &map_idx);
+ if (ret)
+ return ret;
+ }
+
+ epf_dma->num_bar_maps = map_idx;
+
+ ret = pci_epf_dma_build_metadata(epf_dma);
+ if (ret)
+ return ret;
+
+ /* Some DMA resources may already be visible through another map. */
+ for (i = 0; i < epf_dma->num_bar_maps; i++) {
+ if (epf_dma->bar_maps[i].needs_submap)
+ epf_dma->num_submaps++;
+ }
+ if (!epf_dma->num_submaps)
+ return 0;
+
+ dma_window_bar_size = mapped_size;
+ epf_dma->dma_window_addr =
+ pci_epf_alloc_space(epf, dma_window_bar_size,
+ epf_dma->dma_window_bar, epc_features,
+ PRIMARY_INTERFACE);
+ if (!epf_dma->dma_window_addr) {
+ dev_err(dev, "failed to allocate BAR%d DMA window space\n",
+ epf_dma->dma_window_bar);
+ return -ENOMEM;
+ }
+ bar = &epf->bar[epf_dma->dma_window_bar];
+ memset(epf_dma->dma_window_addr, 0, bar->size);
+
+ if (bar->size > mapped_size)
+ epf_dma->num_submaps++;
+
+ epf_dma->submaps = kzalloc_objs(*epf_dma->submaps, epf_dma->num_submaps);
+ if (!epf_dma->submaps)
+ return -ENOMEM;
+
+ for (i = 0; i < epf_dma->num_bar_maps; i++) {
+ if (!epf_dma->bar_maps[i].needs_submap)
+ continue;
+
+ epf_dma->submaps[sub_idx++] = (struct pci_epf_bar_submap) {
+ .phys_addr = epf_dma->bar_maps[i].phys_addr,
+ .size = epf_dma->bar_maps[i].map_size,
+ };
+ }
+
+ /* Cover any BAR tail padding with the allocated scratch space. */
+ if (bar->size > mapped_size) {
+ epf_dma->submaps[sub_idx++] = (struct pci_epf_bar_submap) {
+ .phys_addr = bar->phys_addr + mapped_size,
+ .size = bar->size - mapped_size,
+ };
+ }
+
+ return 0;
+}
+
+static void pci_epf_dma_free_layout(struct pci_epf_dma *epf_dma)
+{
+ struct pci_epf *epf = epf_dma->epf;
+ struct pci_epf_bar *bar;
+
+ if (epf_dma->dma_window_addr) {
+ bar = &epf->bar[epf_dma->dma_window_bar];
+ bar->submap = NULL;
+ bar->num_submap = 0;
+ }
+ epf_dma->submaps_programmed = false;
+
+ kfree(epf_dma->submaps);
+ epf_dma->submaps = NULL;
+ epf_dma->num_submaps = 0;
+
+ kfree(epf_dma->bar_maps);
+ epf_dma->bar_maps = NULL;
+ epf_dma->num_bar_maps = 0;
+
+ pci_epf_dma_release_channels(epf_dma);
+
+ kfree(epf_dma->resources);
+ epf_dma->resources = NULL;
+ epf_dma->num_resources = 0;
+ epf_dma->ctrl = NULL;
+ memset(epf_dma->ep_to_rc_aux_chan, 0, sizeof(epf_dma->ep_to_rc_aux_chan));
+ memset(epf_dma->rc_to_ep_aux_chan, 0, sizeof(epf_dma->rc_to_ep_aux_chan));
+ memset(epf_dma->ep_to_rc_desc, 0, sizeof(epf_dma->ep_to_rc_desc));
+ memset(epf_dma->rc_to_ep_desc, 0, sizeof(epf_dma->rc_to_ep_desc));
+
+ if (epf_dma->dma_window_addr) {
+ pci_epf_free_space(epf, epf_dma->dma_window_addr,
+ epf_dma->dma_window_bar,
+ PRIMARY_INTERFACE);
+ epf_dma->dma_window_addr = NULL;
+ }
+
+ if (epf_dma->metadata_addr) {
+ pci_epf_free_space(epf, epf_dma->metadata_addr,
+ epf_dma->metadata_bar,
+ PRIMARY_INTERFACE);
+ epf_dma->metadata_addr = NULL;
+ }
+ epf_dma->msix_table_offset = 0;
+}
+
+static int pci_epf_dma_program_submaps(struct pci_epf_dma *epf_dma)
+{
+ struct pci_epf *epf = epf_dma->epf;
+ struct pci_epf_bar *bar;
+ int ret;
+
+ if (!epf_dma->dma_window_addr) {
+ pci_epf_dma_set_metadata_ready(epf_dma, true);
+ return 0;
+ }
+
+ if (epf_dma->submaps_programmed)
+ return 0;
+
+ bar = &epf->bar[epf_dma->dma_window_bar];
+ bar->submap = epf_dma->submaps;
+ bar->num_submap = epf_dma->num_submaps;
+
+ ret = pci_epc_set_bar(epf->epc, epf->func_no, epf->vfunc_no, bar);
+ if (ret) {
+ bar->submap = NULL;
+ bar->num_submap = 0;
+ return ret;
+ }
+
+ epf_dma->submaps_programmed = true;
+ pci_epf_dma_set_metadata_ready(epf_dma, true);
+
+ return 0;
+}
+
+static void pci_epf_dma_map_work(struct work_struct *work)
+{
+ struct pci_epf_dma *epf_dma =
+ container_of(to_delayed_work(work), struct pci_epf_dma,
+ map_work);
+ struct pci_epf *epf = epf_dma->epf;
+ int ret;
+
+ if (!epf->epc)
+ return;
+
+ if (!epf->epc->init_complete) {
+ schedule_delayed_work(&epf_dma->map_work,
+ msecs_to_jiffies(PCI_EPF_DMA_HOST_REQ_POLL_MS));
+ return;
+ }
+
+ if (!pci_epf_dma_metadata_host_requested(epf_dma)) {
+ schedule_delayed_work(&epf_dma->map_work,
+ msecs_to_jiffies(PCI_EPF_DMA_HOST_REQ_POLL_MS));
+ return;
+ }
+
+ ret = pci_epf_dma_program_submaps(epf_dma);
+ if (ret)
+ dev_err(&epf->dev, "failed to program DMA window BAR submaps: %d\n",
+ ret);
+}
+
+static int pci_epf_dma_epc_init(struct pci_epf *epf)
+{
+ struct pci_epf_dma *epf_dma = epf_get_drvdata(epf);
+ const struct pci_epc_features *epc_features;
+ struct pci_epc *epc = epf->epc;
+ struct device *dev = &epf->dev;
+ int ret;
+
+ epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
+ if (!epc_features)
+ return -EOPNOTSUPP;
+
+ pci_epf_dma_clear_metadata_status(epf_dma);
+
+ ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no,
+ epf->header);
+ if (ret) {
+ dev_err(dev, "configuration header write failed\n");
+ return ret;
+ }
+
+ ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no,
+ &epf->bar[epf_dma->metadata_bar]);
+ if (ret) {
+ dev_err(dev, "BAR%d setup failed: %d\n",
+ epf_dma->metadata_bar, ret);
+ return ret;
+ }
+
+ if (epf_dma->dma_window_addr) {
+ ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no,
+ &epf->bar[epf_dma->dma_window_bar]);
+ if (ret) {
+ dev_err(dev, "BAR%d setup failed: %d\n",
+ epf_dma->dma_window_bar, ret);
+ goto err_clear_metadata_bar;
+ }
+ }
+
+ if (epc_features->msi_capable && epf->msi_interrupts) {
+ ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no,
+ epf->msi_interrupts);
+ if (ret) {
+ dev_err(dev, "MSI setup failed: %d\n", ret);
+ goto err_clear_dma_window_bar;
+ }
+ }
+
+ if (epc_features->msix_capable && epf->msix_interrupts) {
+ ret = pci_epc_set_msix(epc, epf->func_no, epf->vfunc_no,
+ epf->msix_interrupts,
+ epf_dma->metadata_bar,
+ epf_dma->msix_table_offset);
+ if (ret) {
+ dev_err(dev, "MSI-X setup failed: %d\n", ret);
+ goto err_clear_dma_window_bar;
+ }
+ }
+
+ schedule_delayed_work(&epf_dma->map_work, 0);
+
+ return 0;
+
+err_clear_dma_window_bar:
+ if (epf_dma->dma_window_addr)
+ pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no,
+ &epf->bar[epf_dma->dma_window_bar]);
+err_clear_metadata_bar:
+ pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no,
+ &epf->bar[epf_dma->metadata_bar]);
+ pci_epf_dma_clear_metadata_status(epf_dma);
+
+ return ret;
+}
+
+static void pci_epf_dma_epc_deinit(struct pci_epf *epf)
+{
+ struct pci_epf_dma *epf_dma = epf_get_drvdata(epf);
+ struct pci_epf_bar *bar;
+
+ cancel_delayed_work_sync(&epf_dma->map_work);
+
+ if (!epf_dma->metadata_addr)
+ return;
+
+ pci_epf_dma_clear_metadata_status(epf_dma);
+ if (epf_dma->dma_window_addr) {
+ bar = &epf->bar[epf_dma->dma_window_bar];
+ pci_epc_clear_bar(epf->epc, epf->func_no, epf->vfunc_no, bar);
+ bar->submap = NULL;
+ bar->num_submap = 0;
+ }
+ pci_epc_clear_bar(epf->epc, epf->func_no, epf->vfunc_no,
+ &epf->bar[epf_dma->metadata_bar]);
+ epf_dma->submaps_programmed = false;
+}
+
+static int pci_epf_dma_link_up(struct pci_epf *epf)
+{
+ struct pci_epf_dma *epf_dma = epf_get_drvdata(epf);
+
+ schedule_delayed_work(&epf_dma->map_work, 0);
+
+ return 0;
+}
+
+static int pci_epf_dma_link_down(struct pci_epf *epf)
+{
+ struct pci_epf_dma *epf_dma = epf_get_drvdata(epf);
+
+ cancel_delayed_work_sync(&epf_dma->map_work);
+ pci_epf_dma_set_metadata_ready(epf_dma, false);
+ /*
+ * Link down can invalidate non-sticky inbound ATU state without going
+ * through pci_epc_clear_bar(). Keep the BAR/submap description intact,
+ * but force the next link-up path to reprogram the subrange mappings
+ * for any still-pending host request.
+ */
+ epf_dma->submaps_programmed = false;
+
+ return 0;
+}
+
+static const struct pci_epc_event_ops pci_epf_dma_event_ops = {
+ .epc_init = pci_epf_dma_epc_init,
+ .epc_deinit = pci_epf_dma_epc_deinit,
+ .link_up = pci_epf_dma_link_up,
+ .link_down = pci_epf_dma_link_down,
+};
+
+static int pci_epf_dma_bind(struct pci_epf *epf)
+{
+ struct pci_epf_dma *epf_dma = epf_get_drvdata(epf);
+ const struct pci_epc_features *epc_features;
+ struct pci_epc *epc = epf->epc;
+ bool needs_dma_window;
+ int ret;
+
+ if (WARN_ON_ONCE(!epc))
+ return -EINVAL;
+
+ epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
+ if (!epc_features)
+ return -EOPNOTSUPP;
+
+ if (!epc_features->msi_capable && !epc_features->msix_capable)
+ return -EOPNOTSUPP;
+
+ if ((!epc_features->msi_capable || !epf->msi_interrupts) &&
+ (!epc_features->msix_capable || !epf->msix_interrupts))
+ return -EINVAL;
+
+ ret = pci_epf_dma_collect_resources(epf_dma);
+ if (ret)
+ return ret;
+
+ if (epf_dma->metadata_bar == NO_BAR)
+ epf_dma->metadata_bar =
+ pci_epf_dma_first_usable_bar(epf_dma, epc_features,
+ NO_BAR);
+
+ if (epf_dma->metadata_bar == NO_BAR ||
+ !pci_epf_dma_bar_usable(epc_features, epf_dma->metadata_bar) ||
+ pci_epf_dma_bar_has_fixed_resource(epf_dma, epf_dma->metadata_bar)) {
+ ret = -EINVAL;
+ goto err_free;
+ }
+
+ needs_dma_window = pci_epf_dma_needs_dma_window(epf_dma);
+ if (needs_dma_window) {
+ if (!epc_features->subrange_mapping ||
+ !epc_features->dynamic_inbound_mapping) {
+ ret = -EOPNOTSUPP;
+ goto err_free;
+ }
+
+ if (epf_dma->dma_window_bar == NO_BAR)
+ epf_dma->dma_window_bar =
+ pci_epf_dma_first_usable_bar(epf_dma, epc_features,
+ epf_dma->metadata_bar);
+ if (epf_dma->dma_window_bar == NO_BAR) {
+ ret = -EOPNOTSUPP;
+ goto err_free;
+ }
+ }
+
+ if (epf_dma->dma_window_bar != NO_BAR) {
+ if (!pci_epf_dma_bar_usable(epc_features,
+ epf_dma->dma_window_bar)) {
+ ret = -EINVAL;
+ goto err_free;
+ }
+ if (epf_dma->metadata_bar == epf_dma->dma_window_bar ||
+ pci_epf_dma_bar_has_fixed_resource(epf_dma,
+ epf_dma->dma_window_bar)) {
+ ret = -EINVAL;
+ goto err_free;
+ }
+ }
+
+ ret = pci_epf_dma_build_layout(epf_dma, epc_features);
+ if (ret)
+ goto err_free;
+
+ return 0;
+
+err_free:
+ pci_epf_dma_free_layout(epf_dma);
+
+ return ret;
+}
+
+static void pci_epf_dma_unbind(struct pci_epf *epf)
+{
+ struct pci_epf_dma *epf_dma = epf_get_drvdata(epf);
+
+ cancel_delayed_work_sync(&epf_dma->map_work);
+ if (epf->epc && epf->epc->init_complete)
+ pci_epf_dma_epc_deinit(epf);
+ pci_epf_dma_free_layout(epf_dma);
+}
+
+#define PCI_EPF_DMA_SHOW(_name, _fmt, _val) \
+static ssize_t pci_epf_dma_##_name##_show(struct config_item *item, \
+ char *page) \
+{ \
+ struct config_group *group = to_config_group(item); \
+ struct pci_epf_dma *epf_dma = to_epf_dma(group); \
+ \
+ return sysfs_emit(page, _fmt "\n", (_val)); \
+}
+
+PCI_EPF_DMA_SHOW(metadata_bar, "%d", (int)epf_dma->metadata_bar)
+PCI_EPF_DMA_SHOW(dma_window_bar, "%d", (int)epf_dma->dma_window_bar)
+
+static ssize_t pci_epf_dma_metadata_bar_store(struct config_item *item, const char *page,
+ size_t len)
+{
+ struct config_group *group = to_config_group(item);
+ struct pci_epf_dma *epf_dma = to_epf_dma(group);
+ int bar, ret;
+
+ if (epf_dma->epf->epc)
+ return -EOPNOTSUPP;
+
+ ret = kstrtoint(page, 0, &bar);
+ if (ret)
+ return ret;
+
+ if (bar != NO_BAR && (bar < BAR_0 || bar >= PCI_STD_NUM_BARS))
+ return -EINVAL;
+ if (bar != NO_BAR && bar == epf_dma->dma_window_bar)
+ return -EINVAL;
+
+ epf_dma->metadata_bar = bar;
+
+ return len;
+}
+
+static ssize_t pci_epf_dma_dma_window_bar_store(struct config_item *item,
+ const char *page, size_t len)
+{
+ struct config_group *group = to_config_group(item);
+ struct pci_epf_dma *epf_dma = to_epf_dma(group);
+ int bar, ret;
+
+ if (epf_dma->epf->epc)
+ return -EOPNOTSUPP;
+
+ ret = kstrtoint(page, 0, &bar);
+ if (ret)
+ return ret;
+
+ if (bar != NO_BAR && (bar < BAR_0 || bar >= PCI_STD_NUM_BARS))
+ return -EINVAL;
+ if (bar != NO_BAR && bar == epf_dma->metadata_bar)
+ return -EINVAL;
+
+ epf_dma->dma_window_bar = bar;
+
+ return len;
+}
+
+PCI_EPF_DMA_SHOW(wr_chans, "%u", (unsigned int)epf_dma->wr_chans)
+
+static ssize_t pci_epf_dma_wr_chans_store(struct config_item *item,
+ const char *page, size_t len)
+{
+ struct config_group *group = to_config_group(item);
+ struct pci_epf_dma *epf_dma = to_epf_dma(group);
+ u16 val;
+ int ret;
+
+ if (epf_dma->epf->epc)
+ return -EOPNOTSUPP;
+
+ ret = kstrtou16(page, 0, &val);
+ if (ret)
+ return ret;
+ if (val > EDMA_MAX_WR_CH)
+ return -EINVAL;
+
+ epf_dma->wr_chans = val;
+
+ return len;
+}
+
+PCI_EPF_DMA_SHOW(rd_chans, "%u", (unsigned int)epf_dma->rd_chans)
+
+static ssize_t pci_epf_dma_rd_chans_store(struct config_item *item,
+ const char *page, size_t len)
+{
+ struct config_group *group = to_config_group(item);
+ struct pci_epf_dma *epf_dma = to_epf_dma(group);
+ u16 val;
+ int ret;
+
+ if (epf_dma->epf->epc)
+ return -EOPNOTSUPP;
+
+ ret = kstrtou16(page, 0, &val);
+ if (ret)
+ return ret;
+ if (val > EDMA_MAX_RD_CH)
+ return -EINVAL;
+
+ epf_dma->rd_chans = val;
+
+ return len;
+}
+
+CONFIGFS_ATTR(pci_epf_dma_, metadata_bar);
+CONFIGFS_ATTR(pci_epf_dma_, dma_window_bar);
+CONFIGFS_ATTR(pci_epf_dma_, wr_chans);
+CONFIGFS_ATTR(pci_epf_dma_, rd_chans);
+
+static struct configfs_attribute *pci_epf_dma_attrs[] = {
+ &pci_epf_dma_attr_metadata_bar,
+ &pci_epf_dma_attr_dma_window_bar,
+ &pci_epf_dma_attr_wr_chans,
+ &pci_epf_dma_attr_rd_chans,
+ NULL,
+};
+
+static const struct config_item_type pci_epf_dma_group_type = {
+ .ct_attrs = pci_epf_dma_attrs,
+ .ct_owner = THIS_MODULE,
+};
+
+static struct config_group *pci_epf_dma_add_cfs(struct pci_epf *epf,
+ struct config_group *group)
+{
+ struct pci_epf_dma *epf_dma = epf_get_drvdata(epf);
+ struct config_group *epf_group = &epf_dma->group;
+ struct device *dev = &epf->dev;
+
+ config_group_init_type_name(epf_group, dev_name(dev),
+ &pci_epf_dma_group_type);
+
+ return epf_group;
+}
+
+static const struct pci_epf_device_id pci_epf_dma_ids[] = {
+ {
+ .name = "pci_epf_dma",
+ },
+ {},
+};
+
+static int pci_epf_dma_probe(struct pci_epf *epf,
+ const struct pci_epf_device_id *id)
+{
+ struct pci_epf_dma *epf_dma;
+
+ epf_dma = devm_kzalloc(&epf->dev, sizeof(*epf_dma), GFP_KERNEL);
+ if (!epf_dma)
+ return -ENOMEM;
+
+ epf->header = &pci_epf_dma_header;
+ epf->event_ops = &pci_epf_dma_event_ops;
+
+ epf_dma->epf = epf;
+ epf_dma->metadata_bar = NO_BAR;
+ epf_dma->dma_window_bar = NO_BAR;
+ INIT_DELAYED_WORK(&epf_dma->map_work, pci_epf_dma_map_work);
+
+ epf_set_drvdata(epf, epf_dma);
+
+ return 0;
+}
+
+static const struct pci_epf_ops pci_epf_dma_ops = {
+ .unbind = pci_epf_dma_unbind,
+ .bind = pci_epf_dma_bind,
+ .add_cfs = pci_epf_dma_add_cfs,
+};
+
+static struct pci_epf_driver pci_epf_dma_driver = {
+ .driver.name = "pci_epf_dma",
+ .probe = pci_epf_dma_probe,
+ .id_table = pci_epf_dma_ids,
+ .ops = &pci_epf_dma_ops,
+ .owner = THIS_MODULE,
+};
+
+static int __init pci_epf_dma_init(void)
+{
+ return pci_epf_register_driver(&pci_epf_dma_driver);
+}
+module_init(pci_epf_dma_init);
+
+static void __exit pci_epf_dma_exit(void)
+{
+ pci_epf_unregister_driver(&pci_epf_dma_driver);
+}
+module_exit(pci_epf_dma_exit);
+
+MODULE_DESCRIPTION("PCI EPF DMA DRIVER");
+MODULE_AUTHOR("Koichiro Den <den@valinux.co.jp>");
+MODULE_LICENSE("GPL");
--
2.51.0
^ permalink raw reply related
* [PATCH v3 3/3] Documentation: PCI: Add PCI DMA endpoint function documentation
From: Koichiro Den @ 2026-06-20 17:08 UTC (permalink / raw)
To: Manivannan Sadhasivam, Krzysztof Wilczyński,
Kishon Vijay Abraham I, Bjorn Helgaas, Jonathan Corbet,
Shuah Khan, Vinod Koul, Frank Li, Arnd Bergmann, Damien Le Moal,
Niklas Cassel
Cc: Marek Vasut, Yoshihiro Shimoda, linux-pci, linux-doc,
linux-kernel, dmaengine
In-Reply-To: <20260620170844.3757241-1-den@valinux.co.jp>
Add a function description and a user guide for pci-epf-dma. Describe
the BAR-resident metadata consumed by dw-edma-pcie, the configfs
attributes, endpoint controller requirements and the host-side DMAengine
usage model.
Suggested-by: Randy Dunlap <rdunlap@infradead.org>
Signed-off-by: Koichiro Den <den@valinux.co.jp>
---
Changes in v3:
- Drop trailing colons from subsection headings (Randy).
- Update docs for HDMA native linked-list mode support.
Documentation/PCI/endpoint/index.rst | 2 +
.../PCI/endpoint/pci-dma-function.rst | 188 ++++++++++++++++
Documentation/PCI/endpoint/pci-dma-howto.rst | 201 ++++++++++++++++++
3 files changed, 391 insertions(+)
create mode 100644 Documentation/PCI/endpoint/pci-dma-function.rst
create mode 100644 Documentation/PCI/endpoint/pci-dma-howto.rst
diff --git a/Documentation/PCI/endpoint/index.rst b/Documentation/PCI/endpoint/index.rst
index dd1f62e731c9..cd4107e02ec2 100644
--- a/Documentation/PCI/endpoint/index.rst
+++ b/Documentation/PCI/endpoint/index.rst
@@ -15,6 +15,8 @@ PCI Endpoint Framework
pci-ntb-howto
pci-vntb-function
pci-vntb-howto
+ pci-dma-function
+ pci-dma-howto
pci-nvme-function
function/binding/pci-test
diff --git a/Documentation/PCI/endpoint/pci-dma-function.rst b/Documentation/PCI/endpoint/pci-dma-function.rst
new file mode 100644
index 000000000000..4de02553f5ff
--- /dev/null
+++ b/Documentation/PCI/endpoint/pci-dma-function.rst
@@ -0,0 +1,188 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+================
+PCI DMA Function
+================
+
+:Author: Koichiro Den <den@valinux.co.jp>
+
+The PCI DMA endpoint function exposes an endpoint-integrated DMA controller
+to the PCI host as a PCI DMA controller. A matching host-side driver
+discovers the endpoint DMA metadata and registers the delegated channels with
+the Linux DMAengine framework, so host DMAengine clients can submit
+transfers.
+
+An endpoint Linux system can already use an endpoint-integrated DMA
+controller locally through the normal DMAengine API, for example to transfer
+data between endpoint memory and host addresses reachable over PCI. The PCI
+DMA function provides a different ownership model: it delegates selected
+local DMA channels to the host, so a host DMAengine client can request and
+program those endpoint-side channels through the host's DMAengine API.
+
+To make that possible, the endpoint function publishes the DMA controller
+register window and descriptor memory layout to the host, reserves the
+selected local DMA channels on the endpoint side, and lets the host program
+those channels directly.
+
+Constructs Used for Implementing DMA
+====================================
+
+The PCI DMA function uses the following endpoint-side resources and
+configuration:
+
+ 1) DMA controller register window
+ 2) DMA descriptor memory for endpoint-to-RC channels
+ 3) DMA descriptor memory for RC-to-endpoint channels
+ 4) MSI or MSI-X interrupt vectors selected through configfs
+ 5) One endpoint BAR used to publish metadata
+ 6) If needed, one endpoint BAR used for dynamically mapped DMA windows
+
+The endpoint controller reports the DMA controller register and descriptor
+resources through the endpoint auxiliary resource interface. The PCI DMA
+function uses those descriptions to build the host-visible metadata and to map
+resources that are not already visible to the host.
+
+DMA Controller Register Window
+------------------------------
+
+It contains the DMA controller registers programmed by the host-side driver
+to submit transfers, control channels and handle DMA interrupts.
+
+DMA Descriptor Memory
+---------------------
+
+It contains the descriptor memory used by the DMA controller. The PCI DMA
+function exposes descriptor memory for the delegated endpoint-to-RC and
+RC-to-endpoint channels.
+
+MSI/MSI-X Interrupt Vectors
+---------------------------
+
+They are used by the delegated DMA channels to signal completion and error
+conditions to the host-side driver.
+
+Metadata BAR
+------------
+
+It is the endpoint BAR used to publish the endpoint DMA metadata and handshake
+bits. The BAR remains stable while the endpoint function programs the DMA
+windows.
+
+DMA Window BAR
+--------------
+
+It is the endpoint BAR used for DMA resources that are not already visible
+through a fixed BAR. The endpoint function may switch this BAR to subrange
+mapping after the host-side driver has found the metadata BAR.
+
+BAR Metadata
+============
+
+The endpoint function places a small metadata block at the beginning of the
+selected metadata BAR. The format is defined in
+``include/linux/pci-ep-dma.h``.
+
+The host-side driver scans the function's assigned memory BARs, looks for the
+endpoint DMA metadata magic, requests DMA window programming, waits for the
+READY bit, and then parses the metadata to find the DMA register window and
+descriptor windows.
+
+::
+
+ +----------------------+ metadata BAR offset 0
+ | endpoint DMA metadata|
+ +----------------------+
+ | optional padding |
+ +----------------------+
+
+ +----------------------+ DMA window BAR offset 0
+ | mapped DMA resources |
+ +----------------------+
+ | optional padding |
+ +----------------------+
+
+The metadata can also reference resources that are already host-visible
+through fixed BARs. For example, an endpoint controller may expose the DMA
+controller register window at a fixed BAR offset while descriptor memories
+are mapped into the DMA window BAR by the endpoint function.
+
+The metadata is BAR-resident instead of a self-contained PCI Vendor-Specific
+Extended Capability (VSEC). Some endpoint controllers do not provide writable
+configuration-space backing storage large enough for a new VSEC payload, while
+they can map endpoint memory and controller resources into a BAR.
+
+Channel Ownership
+=================
+
+The ``wr_chans`` attribute exposes endpoint-to-RC DMA write channels. The
+``rd_chans`` attribute exposes RC-to-endpoint DMA read channels. The function
+reserves the selected endpoint-side DMAengine channels so that endpoint-side
+DMAengine clients cannot allocate and use the same hardware channels while
+they are delegated to the host.
+
+The current metadata revision describes channels in dense, zero-based order.
+For example, ``wr_chans = 2`` exposes write channels 0 and 1. Skipping a
+hardware channel in the middle of the exposed range is not supported.
+
+DesignWare eDMA unroll and HDMA compatible layouts require each exposed
+direction to be delegated as a whole. For example, on a controller with two
+write channels, ``wr_chans`` must be either 0 or 2. DesignWare HDMA native
+linked-list mode uses per-channel registers, so a smaller dense prefix can be
+delegated.
+
+Interrupts
+==========
+
+The PCI DMA function exposes DMA interrupts through MSI or MSI-X. The common
+endpoint function ``msi_interrupts`` and ``msix_interrupts`` configfs attributes
+select the interrupt vector counts programmed into endpoint config space. At
+least one MSI or MSI-X vector must be configured before the function is bound
+to an endpoint controller.
+
+Transfer Addressing
+===================
+
+The host-side DMAengine client supplies the endpoint memory address as the
+DMA slave address. For example, the ``dw-edma-pcie`` endpoint DMA metadata
+parser passes that slave address to the DMA controller as a raw endpoint-side
+address instead of translating it through a host PCI BAR resource.
+
+The host memory buffer used as the other side of the transfer is still mapped
+using the normal DMA mapping API on the host.
+
+Endpoint Controller Requirements
+================================
+
+The endpoint controller driver must expose the DMA controller register
+window and per-channel descriptor memories through the endpoint auxiliary
+resource API. Endpoint controllers with other DMA register layouts also need
+matching metadata and host-side DMAengine driver support.
+
+Current DesignWare endpoint DMA support exposes only channels with descriptor
+memory; HDMA native non-linked-list mode is not supported yet.
+
+If any DMA resource is not already host-visible through a fixed BAR, the
+endpoint controller must also support BAR subrange mapping and dynamic inbound
+mapping, because the DMA window BAR is assembled from those resources.
+
+Current Support
+===============
+
+The current host-side support is implemented in ``dw-edma-pcie`` for
+DesignWare eDMA unroll, HDMA compatible and HDMA native linked-list layouts.
+Other PCIe controller DMA implementations need corresponding host-side
+DMAengine driver support.
+
+The ``dw-edma-pcie`` PCI ID table does not contain a generic endpoint DMA PCI
+ID entry. Users need to bind the host-side driver explicitly using
+``driver_override``.
+
+The current metadata revision requires the exposed channels to be a dense
+prefix of the hardware channel numbers.
+
+Security Model
+==============
+
+The interface is intended for trusted endpoint/host deployments. A delegated
+DMA channel can access endpoint memory addresses supplied by a host DMAengine
+client.
diff --git a/Documentation/PCI/endpoint/pci-dma-howto.rst b/Documentation/PCI/endpoint/pci-dma-howto.rst
new file mode 100644
index 000000000000..4bdce63c6f7f
--- /dev/null
+++ b/Documentation/PCI/endpoint/pci-dma-howto.rst
@@ -0,0 +1,201 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+==========================================
+PCI DMA Endpoint Function (EPF) User Guide
+==========================================
+
+:Author: Koichiro Den <den@valinux.co.jp>
+
+This guide shows how to configure the ``pci-epf-dma`` endpoint function driver.
+It uses ``dw-edma-pcie`` as the currently available host-side driver. For the
+hardware model and layout see Documentation/PCI/endpoint/pci-dma-function.rst.
+
+Endpoint Device
+===============
+
+Endpoint Controller Devices
+---------------------------
+
+To find the list of endpoint controller devices in the system::
+
+ # ls /sys/class/pci_epc/
+ e65d0000.pcie-ep
+
+If ``PCI_ENDPOINT_CONFIGFS`` is enabled::
+
+ # ls /sys/kernel/config/pci_ep/controllers
+ e65d0000.pcie-ep
+
+Endpoint Function Drivers
+-------------------------
+
+To find the list of endpoint function drivers in the system::
+
+ # ls /sys/bus/pci-epf/drivers
+ pci_epf_dma pci_epf_test
+
+If ``PCI_ENDPOINT_CONFIGFS`` is enabled::
+
+ # ls /sys/kernel/config/pci_ep/functions
+ pci_epf_dma pci_epf_test
+
+Creating pci-epf-dma Device
+---------------------------
+
+Create a ``pci-epf-dma`` device with configfs::
+
+ # mount -t configfs none /sys/kernel/config
+ # cd /sys/kernel/config/pci_ep/
+ # mkdir functions/pci_epf_dma/dma0
+
+The "mkdir dma0" above creates the ``pci-epf-dma`` function device that will
+be probed by the ``pci_epf_dma`` driver.
+
+The PCI endpoint framework populates the directory with the common
+configurable fields::
+
+ # ls functions/pci_epf_dma/dma0
+ baseclass_code msi_interrupts progif_code subsys_id
+ cache_line_size msix_interrupts revid subsys_vendor_id
+ deviceid pci_epf_dma.0 secondary vendorid
+ interrupt_pin primary subclass_code
+
+The PCI DMA function driver also creates a function-specific sub-directory.
+The numeric suffix depends on the endpoint function instance number::
+
+ # ls functions/pci_epf_dma/dma0/pci_epf_dma.0/
+ dma_window_bar metadata_bar rd_chans wr_chans
+
+Configuring pci-epf-dma Device
+------------------------------
+
+The host-side ``dw-edma-pcie`` PCI ID table does not contain a generic
+endpoint DMA PCI ID entry. Choose a PCI vendor/device ID for the endpoint
+device::
+
+ # echo <vendor-id> > functions/pci_epf_dma/dma0/vendorid
+ # echo <device-id> > functions/pci_epf_dma/dma0/deviceid
+ # echo 1 > functions/pci_epf_dma/dma0/msi_interrupts
+
+The PCI class defaults to ``PCI_BASE_CLASS_SYSTEM`` and
+``PCI_CLASS_SYSTEM_DMA``.
+
+The function-specific attributes are:
+
+============== ============================================================
+Attribute Description
+============== ============================================================
+metadata_bar BAR used to publish the endpoint DMA metadata and handshake
+ bits. It is kept as a stable BAR while the DMA windows are
+ programmed. If this is left unset, the first usable BAR that
+ does not already contain a fixed DMA resource is used.
+dma_window_bar BAR used for DMA resources that are not already host-visible,
+ such as the DMA register window or descriptor windows. This
+ BAR may be switched to subrange mapping after the host driver
+ has found the metadata. If this is left unset and a DMA
+ window is needed, the first usable BAR different from
+ ``metadata_bar`` and not already occupied by a fixed DMA
+ resource is used.
+wr_chans Number of endpoint-to-RC DMA write channels to expose.
+rd_chans Number of RC-to-endpoint DMA read channels to expose.
+============== ============================================================
+
+A sample configuration for a DesignWare eDMA/HDMA compatible controller with
+two write channels and two read channels is given below::
+
+ # echo 0 > functions/pci_epf_dma/dma0/pci_epf_dma.0/metadata_bar
+ # echo 2 > functions/pci_epf_dma/dma0/pci_epf_dma.0/dma_window_bar
+ # echo 2 > functions/pci_epf_dma/dma0/pci_epf_dma.0/wr_chans
+ # echo 2 > functions/pci_epf_dma/dma0/pci_epf_dma.0/rd_chans
+
+``wr_chans`` and ``rd_chans`` default to 0. At least one channel direction
+must be configured. The selected channels are exposed in dense, zero-based
+order; for example, ``wr_chans = 2`` exposes write channels 0 and 1.
+DesignWare eDMA unroll and HDMA compatible layouts require each exposed
+direction to be delegated as a whole, so set a direction to either 0 or the
+number of hardware channels in that direction. DesignWare HDMA native
+linked-list mode allows a smaller dense prefix. If ``dma_window_bar`` is
+configured, it must be different from ``metadata_bar``.
+
+The common ``msi_interrupts`` and ``msix_interrupts`` attributes select the
+number of MSI and MSI-X vectors exposed to the host. At least one MSI or
+MSI-X vector must be configured.
+
+The function-specific attributes can only be changed before the endpoint
+function is bound to an endpoint controller.
+
+Binding pci-epf-dma Device to EP Controller
+-------------------------------------------
+
+The DMA function device should be attached to a PCI endpoint controller
+connected to the host::
+
+ # ln -s controllers/e65d0000.pcie-ep \
+ functions/pci_epf_dma/dma0/primary/
+
+Once the above step is completed, the PCI endpoint controller is ready to
+establish a link with the host.
+
+Start the Link
+--------------
+
+Start the endpoint controller by writing 1 to ``start``::
+
+ # echo 1 > controllers/e65d0000.pcie-ep/start
+
+Root Complex Device
+===================
+
+lspci Output
+------------
+
+Note that the device listed here corresponds to the values populated in the
+endpoint configuration above::
+
+ # lspci -nk
+ 01:00.1 0801: <vendor-id>:<device-id>
+
+If the host was already running while the endpoint function was configured,
+rescan the PCI bus after the endpoint side has completed the configfs setup
+and started the endpoint controller, if the platform supports it.
+
+Bind the endpoint DMA function to ``dw-edma-pcie`` explicitly with
+``driver_override``::
+
+ # modprobe dw_edma_pcie
+ # echo dw-edma-pcie > /sys/bus/pci/devices/0000:01:00.1/driver_override
+ # echo 0000:01:00.1 > /sys/bus/pci/drivers_probe
+
+The device should then be bound to ``dw-edma-pcie``::
+
+ # lspci -nk -s 01:00.1
+ 01:00.1 0801: <vendor-id>:<device-id>
+ Kernel driver in use: dw-edma-pcie
+
+Using pci-epf-dma Device
+------------------------
+
+The host side software uses the standard Linux DMAengine API. A DMAengine
+client driver running on the host must request one of the channels provided by
+``dw-edma-pcie`` and submit a transfer.
+
+For an endpoint-to-RC write transfer, the DMAengine client uses a host DMA
+buffer as the destination and an endpoint-side address as the slave source
+address. For an RC-to-endpoint read transfer, the DMAengine client uses a
+host DMA buffer as the source and an endpoint-side address as the slave
+destination address.
+
+Troubleshooting
+===============
+
+``pci-epf-dma`` requires endpoint controller support for DMA auxiliary
+resources and MSI or MSI-X. If any DMA resource must be mapped dynamically,
+the endpoint controller must also support BAR subrange mapping and dynamic
+inbound mapping. Binding the function to an endpoint controller fails if the
+required capabilities are not available, or if both ``msi_interrupts`` and
+``msix_interrupts`` are zero.
+
+If ``dw-edma-pcie`` fails to probe on the host, check that the endpoint was
+bound to the host driver, that the endpoint BARs were assigned by PCI
+enumeration, and that the endpoint DMA metadata READY bit was set after any
+DMA window BAR submaps were programmed.
--
2.51.0
^ permalink raw reply related
* [stefandoesinger-zx297520:mfd 2/124] htmldocs: Warning: MAINTAINERS references a file that doesn't exist: Documentation/devicetree/zte,zx297520v3-*
From: kernel test robot @ 2026-06-20 17:59 UTC (permalink / raw)
To: Stefan Dösinger ; +Cc: oe-kbuild-all, linux-doc
tree: https://gitlab.com/stefandoesinger/zx297520-kernel mfd
head: c94d760b3ed42a18ecbae9e63d7010c770a8c042
commit: 1e6a3951315be7aa7acb8380b0bb39dde1507e4b [2/124] dt-bindings: clk: zte: Add zx297520v3 top clock and reset bindings
compiler: clang version 22.1.8 (https://github.com/llvm/llvm-project ca7933e47d3a3451d81e72ac174dcb5aa28b59d1)
docutils: docutils (Docutils 0.21.2, Python 3.13.5, on linux)
reproduce: (https://download.01.org/0day-ci/archive/20260620/202606201944.Xa050xUX-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202606201944.Xa050xUX-lkp@intel.com/
All warnings (new ones prefixed by >>):
Warning: Documentation/translations/zh_CN/filesystems/gfs2-uevents.rst references a file that doesn't exist: Documentation/filesystems/gfs2-uevents.rst
Warning: Documentation/translations/zh_CN/filesystems/gfs2.rst references a file that doesn't exist: Documentation/filesystems/gfs2.rst
Warning: Documentation/translations/zh_CN/how-to.rst references a file that doesn't exist: Documentation/xxx/xxx.rst
Warning: Documentation/translations/zh_CN/networking/xfrm_proc.rst references a file that doesn't exist: Documentation/networking/xfrm_proc.rst
Warning: Documentation/translations/zh_CN/scsi/scsi_mid_low_api.rst references a file that doesn't exist: Documentation/Configure.help
>> Warning: MAINTAINERS references a file that doesn't exist: Documentation/devicetree/zte,zx297520v3-*
Warning: MAINTAINERS references a file that doesn't exist: Documentation/ABI/testing/sysfs-platform-ayaneo
Warning: MAINTAINERS references a file that doesn't exist: Documentation/devicetree/bindings/display/bridge/megachips-stdpxxxx-ge-b850v3-fw.txt
Warning: MAINTAINERS references a file that doesn't exist: Documentation/devicetree/bindings/embedded-controller/qcom,hamoa-crd-ec.yaml
Warning: arch/powerpc/sysdev/mpic.c references a file that doesn't exist: Documentation/devicetree/bindings/powerpc/fsl/mpic.txt
Warning: drivers/net/ethernet/smsc/Kconfig references a file that doesn't exist: file:Documentation/networking/device_drivers/ethernet/smsc/smc9.rst
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* Re: [PATCH] docs: ipmi: Fix path of the "hotmod" module parameter
From: Randy Dunlap @ 2026-06-20 18:40 UTC (permalink / raw)
To: Zenghui Yu, openipmi-developer, linux-doc, linux-kernel
Cc: corey, corbet, skhan
In-Reply-To: <20260620122747.7902-1-zenghui.yu@linux.dev>
On 6/20/26 5:27 AM, Zenghui Yu wrote:
> The correct path of the "hotmod" module parameter should be
> /sys/module/ipmi_si/parameters/hotmod. Fix it.
>
> Signed-off-by: Zenghui Yu <zenghui.yu@linux.dev>
> ---
> Documentation/driver-api/ipmi.rst | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
There are several other places that use /sys/modules/ instead of
/sys/module/.
Would you care to fix those also?
thanks.
--
~Randy
^ permalink raw reply
* [PATCH 0/3] Documentation: fix doc build errors for 7.2
From: Rafael Passos @ 2026-06-20 20:16 UTC (permalink / raw)
To: linux-doc; +Cc: corbet, skhan
I noticed these 3 errors when building htmldocs from the mainline
by the end of this merge window.
All are just formatting errors, fixed using the rules from the
"Writing documentation" section in docs.
Thanks,
Rafael Passos (3):
Documentation: iio: fix Malformed table for ltc4283
Documentation: xe_drm: fix chars used for subsection
Documentation: ABI: fix description field indentation
.../ABI/testing/sysfs-class-reboot-mode-reboot_modes | 2 +-
Documentation/hwmon/ltc4283.rst | 4 ++--
include/uapi/drm/xe_drm.h | 6 +++---
3 files changed, 6 insertions(+), 6 deletions(-)
--
2.53.0
^ permalink raw reply
* [PATCH 1/3] Documentation: iio: fix Malformed table for ltc4283
From: Rafael Passos @ 2026-06-20 20:16 UTC (permalink / raw)
To: linux-doc; +Cc: corbet, skhan
In-Reply-To: <20260620201732.94141-1-rafael@rcpassos.me>
Longest line in the first column is 27 chars
Signed-off-by: Rafael Passos <rafael@rcpassos.me>
---
Documentation/hwmon/ltc4283.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Documentation/hwmon/ltc4283.rst b/Documentation/hwmon/ltc4283.rst
index a650c595bc8f..44a58ac6ee81 100644
--- a/Documentation/hwmon/ltc4283.rst
+++ b/Documentation/hwmon/ltc4283.rst
@@ -256,7 +256,7 @@ these logs can be cleared by writing in the proper reset_history attribute.
``/sys/kernel/debug/i2c/i2c-[X]/[X]-addr/``
contains the following attributes:
-======================= ==========================================
+=========================== ===========================================================
power1_failed_fault_log Set to 1 by a power1 fault occurring.
power1_good_input_fault_log Set to 1 by a power1 good input fault occurring at PGIO3.
in11_fet_short_fault_log Set to 1 when a FET-short fault occurs.
@@ -264,4 +264,4 @@ in11_fet_bad_fault_log Set to 1 when a FET-BAD fault occurs.
in0_lcrit_fault_log Set to 1 by a VIN undervoltage fault occurring.
in0_crit_fault_log Set to 1 by a VIN overvoltage fault occurring.
curr1_crit_fault_log Set to 1 by an overcurrent fault occurring.
-======================= ==========================================
+=========================== ===========================================================
--
2.53.0
^ permalink raw reply related
* [PATCH 2/3] Documentation: xe_drm: fix chars used for subsection
From: Rafael Passos @ 2026-06-20 20:16 UTC (permalink / raw)
To: linux-doc; +Cc: corbet, skhan
In-Reply-To: <20260620201732.94141-1-rafael@rcpassos.me>
Equal signs are reserved for document tiles"="
this file docs gets imported by driver-uapi.rst,
and the page title is defined there.
Signed-off-by: Rafael Passos <rafael@rcpassos.me>
---
include/uapi/drm/xe_drm.h | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h
index 48e9f1fdb78d..4dfb30e6c8a8 100644
--- a/include/uapi/drm/xe_drm.h
+++ b/include/uapi/drm/xe_drm.h
@@ -2537,21 +2537,21 @@ struct drm_xe_exec_queue_set_property {
* Refer to Documentation/netlink/specs/drm_ras.yaml for complete interface specification.
*
* Node Registration
- * =================
+ * -----------------
*
* The driver registers DRM RAS nodes for each error severity level.
* enum drm_xe_ras_error_severity defines the node-id, while DRM_XE_RAS_ERROR_SEVERITY_NAMES maps
* node-id to node-name.
*
* Error Classification
- * ====================
+ * --------------------
*
* Each node contains a list of error counters. Each error is identified by a error-id and
* an error-name. enum drm_xe_ras_error_component defines the error-id, while
* DRM_XE_RAS_ERROR_COMPONENT_NAMES maps error-id to error-name.
*
* User Interface
- * ==============
+ * --------------
*
* To retrieve error values of a error counter, userspace applications should
* follow the below steps:
--
2.53.0
^ permalink raw reply related
* [PATCH 3/3] Documentation: ABI: fix description field indentation
From: Rafael Passos @ 2026-06-20 20:16 UTC (permalink / raw)
To: linux-doc; +Cc: corbet, skhan
In-Reply-To: <20260620201732.94141-1-rafael@rcpassos.me>
The "description" field was missaligned.
doc build identifies this as "missing description"
Signed-off-by: Rafael Passos <rafael@rcpassos.me>
---
Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes
index a16c54ab841b..a757a3fe8dd9 100644
--- a/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes
+++ b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes
@@ -2,7 +2,7 @@ What: /sys/class/reboot-mode/<driver>/reboot_modes
Date: March 2026(TBD)
KernelVersion: TBD
Contact: linux-pm@vger.kernel.org
- Description:
+Description:
This interface exposes the reboot-mode arguments
registered with the reboot-mode framework. It is
a read-only interface and provides a space
--
2.53.0
^ permalink raw reply related
* Re: [PATCH 1/3] Documentation: iio: fix Malformed table for ltc4283
From: Randy Dunlap @ 2026-06-20 21:29 UTC (permalink / raw)
To: Rafael Passos, linux-doc; +Cc: corbet, skhan
In-Reply-To: <20260620201732.94141-2-rafael@rcpassos.me>
Hi,
On 6/20/26 1:16 PM, Rafael Passos wrote:
> Longest line in the first column is 27 chars
>
> Signed-off-by: Rafael Passos <rafael@rcpassos.me>
> ---
> Documentation/hwmon/ltc4283.rst | 4 ++--
> 1 file changed, 2 insertions(+), 2 deletions(-)
Why "iio:" in the Subject line instead of "hwmon:"?
I sent this patch yesterday (my local time):
https://lore.kernel.org/linux-doc/20260620011833.3568693-1-rdunlap@infradead.org/T/#u
scripts/get_maintainer.pl should have told you to send the patch the the hwmon
mailing list and the HWMON maintainer (as well as Documentation).
>
> diff --git a/Documentation/hwmon/ltc4283.rst b/Documentation/hwmon/ltc4283.rst
> index a650c595bc8f..44a58ac6ee81 100644
> --- a/Documentation/hwmon/ltc4283.rst
> +++ b/Documentation/hwmon/ltc4283.rst
> @@ -256,7 +256,7 @@ these logs can be cleared by writing in the proper reset_history attribute.
> ``/sys/kernel/debug/i2c/i2c-[X]/[X]-addr/``
> contains the following attributes:
>
> -======================= ==========================================
> +=========================== ===========================================================
> power1_failed_fault_log Set to 1 by a power1 fault occurring.
> power1_good_input_fault_log Set to 1 by a power1 good input fault occurring at PGIO3.
> in11_fet_short_fault_log Set to 1 when a FET-short fault occurs.
> @@ -264,4 +264,4 @@ in11_fet_bad_fault_log Set to 1 when a FET-BAD fault occurs.
> in0_lcrit_fault_log Set to 1 by a VIN undervoltage fault occurring.
> in0_crit_fault_log Set to 1 by a VIN overvoltage fault occurring.
> curr1_crit_fault_log Set to 1 by an overcurrent fault occurring.
> -======================= ==========================================
> +=========================== ===========================================================
--
~Randy
^ permalink raw reply
* Re: [PATCH 3/3] Documentation: ABI: fix description field indentation
From: Randy Dunlap @ 2026-06-20 21:33 UTC (permalink / raw)
To: Rafael Passos, linux-doc; +Cc: corbet, skhan
In-Reply-To: <20260620201732.94141-4-rafael@rcpassos.me>
On 6/20/26 1:16 PM, Rafael Passos wrote:
> The "description" field was missaligned.
> doc build identifies this as "missing description"
>
> Signed-off-by: Rafael Passos <rafael@rcpassos.me>
> ---
> Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes
> index a16c54ab841b..a757a3fe8dd9 100644
> --- a/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes
> +++ b/Documentation/ABI/testing/sysfs-class-reboot-mode-reboot_modes
> @@ -2,7 +2,7 @@ What: /sys/class/reboot-mode/<driver>/reboot_modes
> Date: March 2026(TBD)
> KernelVersion: TBD
> Contact: linux-pm@vger.kernel.org
> - Description:
> +Description:
> This interface exposes the reboot-mode arguments
> registered with the reboot-mode framework. It is
> a read-only interface and provides a space
This and other issues in this file are already fixed here:
https://lore.kernel.org/all/178130191372.340022.764793265726304664.b4-ty@collabora.com/
--
~Randy
^ permalink raw reply
* Re: [PATCH 2/3] Documentation: xe_drm: fix chars used for subsection
From: Randy Dunlap @ 2026-06-20 21:42 UTC (permalink / raw)
To: Rafael Passos, linux-doc; +Cc: corbet, skhan
In-Reply-To: <20260620201732.94141-3-rafael@rcpassos.me>
Hi,
On 6/20/26 1:16 PM, Rafael Passos wrote:
> Equal signs are reserved for document tiles"="
titles.
> this file docs gets imported by driver-uapi.rst,
This
> and the page title is defined there.
It would be helpful to include the warnings here (but maybe not
all 10 lines of each warning).
This patch does indeed prevent the warnings.
You should send this patch the the DRM XE maintainers & mailing list.
INTEL DRM DISPLAY FOR XE AND I915 DRIVERS
M: Jani Nikula <jani.nikula@linux.intel.com>
M: Rodrigo Vivi <rodrigo.vivi@intel.com>
L: intel-gfx@lists.freedesktop.org
L: intel-xe@lists.freedesktop.org
scripts/get_maintainer.pl should have told you that. (It does for me.)
Thanks.
>
> Signed-off-by: Rafael Passos <rafael@rcpassos.me>
> ---
> include/uapi/drm/xe_drm.h | 6 +++---
> 1 file changed, 3 insertions(+), 3 deletions(-)
>
> diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h
> index 48e9f1fdb78d..4dfb30e6c8a8 100644
> --- a/include/uapi/drm/xe_drm.h
> +++ b/include/uapi/drm/xe_drm.h
> @@ -2537,21 +2537,21 @@ struct drm_xe_exec_queue_set_property {
> * Refer to Documentation/netlink/specs/drm_ras.yaml for complete interface specification.
> *
> * Node Registration
> - * =================
> + * -----------------
> *
> * The driver registers DRM RAS nodes for each error severity level.
> * enum drm_xe_ras_error_severity defines the node-id, while DRM_XE_RAS_ERROR_SEVERITY_NAMES maps
> * node-id to node-name.
> *
> * Error Classification
> - * ====================
> + * --------------------
> *
> * Each node contains a list of error counters. Each error is identified by a error-id and
> * an error-name. enum drm_xe_ras_error_component defines the error-id, while
> * DRM_XE_RAS_ERROR_COMPONENT_NAMES maps error-id to error-name.
> *
> * User Interface
> - * ==============
> + * --------------
> *
> * To retrieve error values of a error counter, userspace applications should
> * follow the below steps:
--
~Randy
^ permalink raw reply
* Re: [PATCH] docs: ipmi: Fix path of the "hotmod" module parameter
From: Zenghui Yu @ 2026-06-20 23:06 UTC (permalink / raw)
To: Randy Dunlap
Cc: openipmi-developer, linux-doc, linux-kernel, corey, corbet, skhan
In-Reply-To: <626477f6-8bda-4cac-8341-c720fd279ba3@infradead.org>
On 6/21/26 2:40 AM, Randy Dunlap wrote:
>
>
> On 6/20/26 5:27 AM, Zenghui Yu wrote:
> > The correct path of the "hotmod" module parameter should be
> > /sys/module/ipmi_si/parameters/hotmod. Fix it.
> >
> > Signed-off-by: Zenghui Yu <zenghui.yu@linux.dev>
> > ---
> > Documentation/driver-api/ipmi.rst | 2 +-
> > 1 file changed, 1 insertion(+), 1 deletion(-)
>
> There are several other places that use /sys/modules/ instead of
> /sys/module/.
Yup. There are:
Documentation/driver-api/ipmi.rst:This is done using /sys/modules/ipmi_si/parameters/hotmod, which is a
Documentation/process/debugging/kgdb.rst:config string to ``/sys/module/<driver>/parameter/<option>``. The driver
Documentation/translations/zh_CN/admin-guide/mm/damon/lru_sort.rst:参数，或者在 ``/sys/modules/damon_lru_sort/parameters/<parameter>`` 写入正确的
Documentation/translations/zh_CN/admin-guide/mm/damon/lru_sort.rst: # cd /sys/modules/damon_lru_sort/parameters
Documentation/translations/zh_TW/admin-guide/mm/damon/lru_sort.rst:參數，或者在 ``/sys/modules/damon_lru_sort/parameters/<parameter>`` 寫入正確的
Documentation/translations/zh_TW/admin-guide/mm/damon/lru_sort.rst: # cd /sys/modules/damon_lru_sort/parameters
drivers/acpi/sysfs.c: * /sys/modules/acpi/parameters/debug_layer
drivers/acpi/sysfs.c: * /sys/modules/acpi/parameters/debug_level
drivers/acpi/sysfs.c: * /sys/modules/acpi/parameters/trace_method_name
drivers/acpi/sysfs.c: * /sys/modules/acpi/parameters/trace_state
drivers/acpi/sysfs.c: * /sys/modules/acpi/parameters/trace_debug_layer
drivers/acpi/sysfs.c: * /sys/modules/acpi/parameters/trace_debug_level
drivers/acpi/sysfs.c:/* /sys/modules/acpi/parameters/aml_debug_output */
drivers/base/module.c: /* Lookup or create built-in module entry in /sys/modules */
drivers/gpib/lpvo_usb_gpib/lpvo_usb_gpib.c: * On the fly: echo {0,1,2} > /sys/modules/lpvo_usb_gpib/parameters/debug
fs/btrfs/sysfs.c:/* Set perms to 0, disable /sys/module/btrfs/parameter/read_policy interface. */
fs/cachefiles/Kconfig: enabled by setting bits in /sys/modules/cachefiles/parameter/debug or
kernel/params.c:/* sysfs output in /sys/modules/XYZ/parameters/ */
>
> Would you care to fix those also?
I plan to fix them by subsystem like:
https://lore.kernel.org/20260611142518.77343-1-zenghui.yu@linux.dev
Thanks,
Zenghui
^ permalink raw reply
* [PATCH] docs: kgdb: Fix path of driver options
From: Zenghui Yu @ 2026-06-20 23:40 UTC (permalink / raw)
To: kgdb-bugreport, workflows, linux-doc, linux-kernel
Cc: jason.wessel, danielt, dianders, corbet, skhan, rdunlap,
Zenghui Yu
The correct path of driver options should be
/sys/module/<driver>/parameters/<option>. Fix it.
Signed-off-by: Zenghui Yu <zenghui.yu@linux.dev>
---
Documentation/process/debugging/kgdb.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Documentation/process/debugging/kgdb.rst b/Documentation/process/debugging/kgdb.rst
index c4d0a9121d52..316b1d74e9c8 100644
--- a/Documentation/process/debugging/kgdb.rst
+++ b/Documentation/process/debugging/kgdb.rst
@@ -513,7 +513,7 @@ unregister all the kernel hook points.
All kgdb I/O drivers can be reconfigured at run time, if
``CONFIG_SYSFS`` and ``CONFIG_MODULES`` are enabled, by echo'ing a new
-config string to ``/sys/module/<driver>/parameter/<option>``. The driver
+config string to ``/sys/module/<driver>/parameters/<option>``. The driver
can be unconfigured by passing an empty string. You cannot change the
configuration while the debugger is attached. Make sure to detach the
debugger with the ``detach`` command prior to trying to unconfigure a
--
2.53.0
^ permalink raw reply related
* Re: [PATCH v6 3/6] alloc_tag: add size-based filtering to ioctl
From: Suren Baghdasaryan @ 2026-06-21 0:00 UTC (permalink / raw)
To: Abhishek Bapat
Cc: Andrew Morton, Kent Overstreet, Hao Ge, Shuah Khan,
Jonathan Corbet, linux-doc, linux-kernel, linux-mm, Sourav Panda
In-Reply-To: <6944ab65167d8884ce0d856184730d06ead68cb5.1781803482.git.abhishekbapat@google.com>
On Thu, Jun 18, 2026 at 10:36 AM Abhishek Bapat
<abhishekbapat@google.com> wrote:
>
> Extend the allocinfo filtering mechanism to allow users to filter tags
> based on the total number of bytes allocated [min_size, max_size]. The
> size range is inclusive.
>
> Filtering by size involves retrieving allocinfo per-CPU counters, which
> is an expensive operation. Hence, the performance of size-based
> filtering will be worse than other filters.
>
> Signed-off-by: Abhishek Bapat <abhishekbapat@google.com>
> Acked-by: Hao Ge <hao.ge@linux.dev>
Acked-by: Suren Baghdasaryan <surenb@google.com>
> ---
> include/uapi/linux/alloc_tag.h | 8 ++++-
> lib/alloc_tag.c | 64 +++++++++++++++++++++++++++-------
> 2 files changed, 58 insertions(+), 14 deletions(-)
>
> diff --git a/include/uapi/linux/alloc_tag.h b/include/uapi/linux/alloc_tag.h
> index 13e9b5916bf5..0de5fc180790 100644
> --- a/include/uapi/linux/alloc_tag.h
> +++ b/include/uapi/linux/alloc_tag.h
> @@ -50,13 +50,17 @@ enum {
> ALLOCINFO_FILTER_FUNCTION,
> ALLOCINFO_FILTER_FILENAME,
> ALLOCINFO_FILTER_LINENO,
> - __ALLOCINFO_FILTER_LAST = ALLOCINFO_FILTER_LINENO
> + ALLOCINFO_FILTER_MIN_SIZE,
> + ALLOCINFO_FILTER_MAX_SIZE,
> + __ALLOCINFO_FILTER_LAST = ALLOCINFO_FILTER_MAX_SIZE
> };
>
> #define ALLOCINFO_FILTER_MASK_MODNAME (1 << ALLOCINFO_FILTER_MODNAME)
> #define ALLOCINFO_FILTER_MASK_FUNCTION (1 << ALLOCINFO_FILTER_FUNCTION)
> #define ALLOCINFO_FILTER_MASK_FILENAME (1 << ALLOCINFO_FILTER_FILENAME)
> #define ALLOCINFO_FILTER_MASK_LINENO (1 << ALLOCINFO_FILTER_LINENO)
> +#define ALLOCINFO_FILTER_MASK_MIN_SIZE (1 << ALLOCINFO_FILTER_MIN_SIZE)
> +#define ALLOCINFO_FILTER_MASK_MAX_SIZE (1 << ALLOCINFO_FILTER_MAX_SIZE)
>
> #define ALLOCINFO_FILTER_MASKS \
> ((1 << (__ALLOCINFO_FILTER_LAST + 1)) - 1)
> @@ -64,6 +68,8 @@ enum {
> struct allocinfo_filter {
> __u64 mask; /* bitmask of the filter fields used */
> struct allocinfo_tag fields;
> + __u64 min_size;
> + __u64 max_size;
> };
>
> struct allocinfo_get_at {
> diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
> index f00d731b81cf..ad33d63ef7b4 100644
> --- a/lib/alloc_tag.c
> +++ b/lib/alloc_tag.c
> @@ -198,16 +198,20 @@ static int allocinfo_cmp_str(const char *str, const char *template)
> return strncmp(allocinfo_str(str), template, ALLOCINFO_STR_SIZE);
> }
>
> +/* Fetch the per-CPU counters */
> +static inline struct alloc_tag_counters allocinfo_prefetch_counters(struct codetag *ct)
> +{
> + return alloc_tag_read(ct_to_alloc_tag(ct));
> +}
> +
> /*
> * Populates the UAPI allocinfo_tag_data structure with active runtime
> * profiling counters extracted from the given kernel codetag.
> */
> static void allocinfo_to_params(struct codetag *ct,
> - struct allocinfo_tag_data *data)
> + struct allocinfo_tag_data *data,
> + struct alloc_tag_counters *counters)
> {
> - struct alloc_tag *tag = ct_to_alloc_tag(ct);
> - struct alloc_tag_counters counter = alloc_tag_read(tag);
> -
> if (ct->modname)
> allocinfo_copy_str(data->tag.modname, ct->modname);
> else
> @@ -215,9 +219,9 @@ static void allocinfo_to_params(struct codetag *ct,
> allocinfo_copy_str(data->tag.function, ct->function);
> allocinfo_copy_str(data->tag.filename, ct->filename);
> data->tag.lineno = ct->lineno;
> - data->counter.bytes = counter.bytes;
> - data->counter.calls = counter.calls;
> - data->counter.accurate = !alloc_tag_is_inaccurate(tag);
> + data->counter.bytes = counters->bytes;
> + data->counter.calls = counters->calls;
> + data->counter.accurate = !alloc_tag_is_inaccurate(ct_to_alloc_tag(ct));
> }
>
> /*
> @@ -241,7 +245,9 @@ static int allocinfo_ioctl_get_content_id(struct seq_file *m, void __user *arg)
> * Verifies whether a given codetag satisfies the active filtering criteria by
> * matching its characteristics against the specified filter.
> */
> -static bool matches_filter(struct codetag *ct, struct allocinfo_filter *filter)
> +static bool matches_filter(struct codetag *ct, struct allocinfo_filter *filter,
> + struct alloc_tag_counters *counters,
> + bool *fetched_counters)
> {
> if (!filter || !filter->mask)
> return true;
> @@ -268,6 +274,19 @@ static bool matches_filter(struct codetag *ct, struct allocinfo_filter *filter)
> ct->lineno != filter->fields.lineno)
> return false;
>
> + if (filter->mask & (ALLOCINFO_FILTER_MASK_MIN_SIZE | ALLOCINFO_FILTER_MASK_MAX_SIZE)) {
> + if (!*fetched_counters) {
> + *counters = allocinfo_prefetch_counters(ct);
> + *fetched_counters = true;
> + }
> + if ((filter->mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) &&
> + counters->bytes < filter->min_size)
> + return false;
> + if ((filter->mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) &&
> + counters->bytes > filter->max_size)
> + return false;
> + }
> +
> return true;
> }
>
> @@ -281,6 +300,8 @@ static int allocinfo_ioctl_get_at(struct seq_file *m, void __user *arg)
> struct codetag *ct;
> struct allocinfo_get_at params = {0};
> __u64 skip_count;
> + struct alloc_tag_counters counters;
> + bool fetched_counters;
>
> if (copy_from_user(¶ms, arg, sizeof(params)))
> return -EFAULT;
> @@ -288,6 +309,11 @@ static int allocinfo_ioctl_get_at(struct seq_file *m, void __user *arg)
> if (params.filter.mask & ~ALLOCINFO_FILTER_MASKS)
> return -EINVAL;
>
> + if ((params.filter.mask & ALLOCINFO_FILTER_MASK_MIN_SIZE) &&
> + (params.filter.mask & ALLOCINFO_FILTER_MASK_MAX_SIZE) &&
> + params.filter.min_size > params.filter.max_size)
> + return -EINVAL;
> +
> priv = m->private;
>
> mutex_lock(&priv->ioctl_lock);
> @@ -311,7 +337,8 @@ static int allocinfo_ioctl_get_at(struct seq_file *m, void __user *arg)
> ct = codetag_next_ct(&priv->ioctl_iter);
>
> while (ct) {
> - if (matches_filter(ct, &priv->filter)) {
> + fetched_counters = false;
> + if (matches_filter(ct, &priv->filter, &counters, &fetched_counters)) {
> if (skip_count == 0)
> break;
> skip_count--;
> @@ -320,7 +347,9 @@ static int allocinfo_ioctl_get_at(struct seq_file *m, void __user *arg)
> }
>
> if (ct) {
> - allocinfo_to_params(ct, ¶ms.data);
> + if (!fetched_counters)
> + counters = allocinfo_prefetch_counters(ct);
> + allocinfo_to_params(ct, ¶ms.data, &counters);
> priv->positioned = true;
> }
>
> @@ -346,6 +375,8 @@ static int allocinfo_ioctl_get_next(struct seq_file *m, void __user *arg)
> struct codetag *ct;
> struct allocinfo_tag_data params;
> int ret = 0;
> + struct alloc_tag_counters counters;
> + bool fetched_counters;
>
> memset(¶ms, 0, sizeof(params));
> priv = m->private;
> @@ -359,11 +390,18 @@ static int allocinfo_ioctl_get_next(struct seq_file *m, void __user *arg)
> }
>
> ct = codetag_next_ct(&priv->ioctl_iter);
> - while (ct && !matches_filter(ct, &priv->filter))
> + while (ct) {
> + fetched_counters = false;
> + if (matches_filter(ct, &priv->filter, &counters, &fetched_counters))
> + break;
> ct = codetag_next_ct(&priv->ioctl_iter);
> - if (ct)
> - allocinfo_to_params(ct, ¶ms);
> + }
>
> + if (ct) {
> + if (!fetched_counters)
> + counters = allocinfo_prefetch_counters(ct);
> + allocinfo_to_params(ct, ¶ms, &counters);
> + }
> if (!ct) {
> priv->positioned = false;
> ret = -ENOENT;
> --
> 2.55.0.rc0.786.g65d90a0328-goog
>
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox