From: Dapeng Mi <dapeng1.mi@linux.intel.com>
To: Peter Zijlstra <peterz@infradead.org>,
Ingo Molnar <mingo@redhat.com>,
Arnaldo Carvalho de Melo <acme@kernel.org>,
Namhyung Kim <namhyung@kernel.org>,
Ian Rogers <irogers@google.com>,
Adrian Hunter <adrian.hunter@intel.com>,
Alexander Shishkin <alexander.shishkin@linux.intel.com>,
Andi Kleen <ak@linux.intel.com>,
Eranian Stephane <eranian@google.com>,
Sandipan Das <sandipan.das@amd.com>,
Andrii Nakryiko <andrii@kernel.org>, Song Liu <song@kernel.org>,
Ravi Bangoria <ravi.bangoria@amd.com>,
Ananth Narayan <ananth.narayan@amd.com>
Cc: linux-kernel@vger.kernel.org, linux-perf-users@vger.kernel.org,
bpf@vger.kernel.org, Dapeng Mi <dapeng1.mi@intel.com>,
Zide Chen <zide.chen@intel.com>,
Falcon Thomas <thomas.falcon@intel.com>,
Xudong Hao <xudong.hao@intel.com>,
Dapeng Mi <dapeng1.mi@linux.intel.com>
Subject: [Patch v6 11/20] perf/x86/intel: Allow LBR read helpers to operate on explicit stacks
Date: Fri, 6 Nov 2026 12:32:48 +0800 [thread overview]
Message-ID: <20261106043257.842864-12-dapeng1.mi@linux.intel.com> (raw)
In-Reply-To: <20261106043257.842864-1-dapeng1.mi@linux.intel.com>
The LBR read helpers currently assume the default stack,
cpuc->lbr_stack, and therefore cannot be reused for other LBR
stacks that may be in use. Extend the *_lbr_read() helpers to take
an explicit LBR stack argument, so they can operate on any LBR stack
instead of only the default one.
Update the related callers, including intel_pmu_store_lbr() and
intel_pmu_lbr_filter(), to pass the appropriate stack explicitly.
No function change intended.
Fixes: c22ac2a3d4bd ("perf: Enable branch record for software events")
Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
---
arch/x86/events/amd/brs.c | 2 +-
arch/x86/events/amd/lbr.c | 2 +-
arch/x86/events/intel/core.c | 4 +-
arch/x86/events/intel/lbr.c | 89 ++++++++++++++++++++----------------
arch/x86/events/perf_event.h | 16 ++++---
5 files changed, 64 insertions(+), 49 deletions(-)
diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c
index 1906d1257646..475e3699d3ca 100644
--- a/arch/x86/events/amd/brs.c
+++ b/arch/x86/events/amd/brs.c
@@ -316,7 +316,7 @@ void amd_brs_drain(void)
num = tos - start + 1;
- cpu_init_lbr_stack(cpuc, num);
+ cpu_init_lbr_stack(cpu_lbr_stack(cpuc), num);
/*
* BRS is only one pass (saturation) from MSROFF to depth-1
diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c
index 3e4e5e984649..95898a1d9901 100644
--- a/arch/x86/events/amd/lbr.c
+++ b/arch/x86/events/amd/lbr.c
@@ -179,7 +179,7 @@ void amd_pmu_lbr_read(void)
return;
}
- cpu_init_lbr_stack(cpuc, x86_pmu.lbr_nr);
+ cpu_init_lbr_stack(cpu_lbr_stack(cpuc), x86_pmu.lbr_nr);
for (i = 0; i < x86_pmu.lbr_nr; i++) {
entry.from.full = amd_pmu_lbr_get_from(i);
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 42d4d525a729..b7045c05b736 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -2826,7 +2826,7 @@ __intel_pmu_snapshot_branch_stack(struct perf_branch_entry *entries,
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
- intel_pmu_lbr_read();
+ intel_pmu_lbr_read(cpu_lbr_stack(cpuc), cpuc->lbr_counters);
cnt = min_t(unsigned int, cnt, cpu_lbr_stack(cpuc)->nr);
memcpy(entries, cpu_lbr_stack(cpuc)->entries,
@@ -4062,7 +4062,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
loops = 0;
again:
- intel_pmu_lbr_read();
+ intel_pmu_lbr_read(cpu_lbr_stack(cpuc), cpuc->lbr_counters);
/*
* Do not clear the LBR frozen status here; keep LBR frozen
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index 62f2f4b9cfde..979b094ae228 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -105,7 +105,9 @@
#define ARCH_LBR_CTL_MASK 0x7f000e
-static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
+static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack,
+ u64 *lbr_counters);
static __always_inline bool is_lbr_call_stack_bit_set(u64 config)
{
@@ -741,13 +743,14 @@ void intel_pmu_lbr_disable_all(void)
}
}
-void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
+void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack)
{
unsigned long mask = x86_pmu.lbr_nr - 1;
u64 tos = intel_pmu_lbr_tos();
int i;
- cpu_init_lbr_stack(cpuc, x86_pmu.lbr_nr);
+ cpu_init_lbr_stack(lbr_stack, x86_pmu.lbr_nr);
for (i = 0; i < x86_pmu.lbr_nr; i++) {
unsigned long lbr_idx = (tos - i) & mask;
union {
@@ -760,13 +763,13 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
- cpu_lbr_stack(cpuc)->entries[i] = (struct perf_branch_entry){
+ lbr_stack->entries[i] = (struct perf_branch_entry){
.from = msr_lastbranch.from,
.to = msr_lastbranch.to,
};
}
- cpu_lbr_stack(cpuc)->nr = i;
- cpu_lbr_stack(cpuc)->hw_idx = tos;
+ lbr_stack->nr = i;
+ lbr_stack->hw_idx = tos;
}
/*
@@ -774,7 +777,8 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
* is the same as the linear address, allowing us to merge the LIP and EIP
* LBR formats.
*/
-void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
+void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack)
{
bool need_info = false, call_stack = false;
unsigned long mask = x86_pmu.lbr_nr - 1;
@@ -789,7 +793,7 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
call_stack = true;
}
- cpu_init_lbr_stack(cpuc, num);
+ cpu_init_lbr_stack(lbr_stack, num);
for (i = 0; i < num; i++) {
unsigned long lbr_idx = (tos - i) & mask;
u64 from, to, mis = 0, pred = 0, in_tx = 0, abort = 0;
@@ -850,7 +854,7 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
if (abort && x86_pmu.lbr_double_abort && out > 0)
out--;
- cpu_lbr_stack(cpuc)->entries[out] = (struct perf_branch_entry){
+ lbr_stack->entries[out] = (struct perf_branch_entry){
.from = from,
.to = to,
.mispred = mis,
@@ -861,8 +865,8 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
};
out++;
}
- cpu_lbr_stack(cpuc)->nr = out;
- cpu_lbr_stack(cpuc)->hw_idx = tos;
+ lbr_stack->nr = out;
+ lbr_stack->hw_idx = tos;
}
static DEFINE_STATIC_KEY_FALSE(x86_lbr_mispred);
@@ -903,17 +907,19 @@ static __always_inline u16 get_lbr_cycles(u64 info)
static_assert((64 - PERF_BRANCH_ENTRY_INFO_BITS_MAX) > LBR_INFO_BR_CNTR_NUM * LBR_INFO_BR_CNTR_BITS);
-static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
- struct lbr_entry *entries)
+static void intel_pmu_store_lbr(struct lbr_entry *entries,
+ struct perf_branch_stack *lbr_stack)
{
+ struct perf_branch_entry *e;
struct lbr_entry *lbr;
u64 from, to, info;
bool mispred;
int i;
- cpu_init_lbr_stack(cpuc, x86_pmu.lbr_nr);
+ cpu_init_lbr_stack(lbr_stack, x86_pmu.lbr_nr);
for (i = 0; i < x86_pmu.lbr_nr; i++) {
lbr = entries ? &entries[i] : NULL;
+ e = &lbr_stack->entries[i];
from = rdlbr_from(i, lbr);
/*
@@ -927,7 +933,7 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
mispred = get_lbr_mispred(info);
- cpu_lbr_stack(cpuc)->entries[i] = (struct perf_branch_entry){
+ *e = (struct perf_branch_entry){
.from = from,
.to = to,
.mispred = mispred,
@@ -948,7 +954,7 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc,
};
}
- cpu_lbr_stack(cpuc)->nr = i;
+ lbr_stack->nr = i;
}
/*
@@ -999,37 +1005,41 @@ void intel_pmu_lbr_save_brstack(struct perf_sample_data *data,
perf_sample_save_brstack(data, event, cpu_lbr_stack(cpuc), NULL);
}
-static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc)
+static void intel_pmu_arch_lbr_read(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack)
{
- intel_pmu_store_lbr(cpuc, NULL);
+ intel_pmu_store_lbr(NULL, lbr_stack);
}
-static void intel_pmu_arch_lbr_read_xsave(struct cpu_hw_events *cpuc)
+static void
+intel_pmu_arch_lbr_read_xsave(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack)
{
struct x86_perf_task_context_arch_lbr_xsave *xsave = cpuc->lbr_xsave;
if (!xsave) {
- intel_pmu_store_lbr(cpuc, NULL);
+ intel_pmu_store_lbr(NULL, lbr_stack);
return;
}
xsave->header.xfeatures = 0;
xsaves(&xsave->xsave, XFEATURE_MASK_LBR);
if (xsave->header.xfeatures & XFEATURE_MASK_LBR) {
- intel_pmu_store_lbr(cpuc, xsave->lbr.entries);
+ intel_pmu_store_lbr(xsave->lbr.entries, lbr_stack);
} else {
/* Clear the LBR stack to avoid leaking stale entries. */
- cpu_lbr_stack(cpuc)->nr = 0;
+ lbr_stack->nr = 0;
}
}
-void intel_pmu_lbr_read(void)
+void intel_pmu_lbr_read(struct perf_branch_stack *lbr_stack,
+ u64 *lbr_counters)
{
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
if (!cpuc->lbr_users || vlbr_exclude_host()) {
- /* Reset lbr_stack.nr to prevent stale data leakage. */
- cpu_lbr_stack(cpuc)->nr = 0;
+ /* Reset lbr_stack->nr to prevent stale data leakage. */
+ lbr_stack->nr = 0;
return;
}
@@ -1042,9 +1052,9 @@ void intel_pmu_lbr_read(void)
if (cpuc->lbr_users == cpuc->lbr_pebs_users)
return;
- x86_pmu.lbr_read(cpuc);
+ x86_pmu.lbr_read(cpuc, lbr_stack);
- intel_pmu_lbr_filter(cpuc);
+ intel_pmu_lbr_filter(cpuc, lbr_stack, lbr_counters);
}
/*
@@ -1227,7 +1237,9 @@ static const int arch_lbr_br_type_map[ARCH_LBR_BR_TYPE_MAP_MAX] = {
* in PERF_SAMPLE_BRANCH_STACK sample may vary.
*/
static void
-intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
+intel_pmu_lbr_filter(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack,
+ u64 *lbr_counters)
{
int br_sel = cpuc->br_sel;
int i, j, type, from_plm, to_plm;
@@ -1240,8 +1252,8 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
((br_sel & X86_BR_TYPE_SAVE) != X86_BR_TYPE_SAVE))
return;
- for (i = 0; i < cpu_lbr_stack(cpuc)->nr; i++) {
- struct perf_branch_entry *be = &cpu_lbr_stack(cpuc)->entries[i];
+ for (i = 0; i < lbr_stack->nr; i++) {
+ struct perf_branch_entry *be = &lbr_stack->entries[i];
from = be->from;
to = be->to;
@@ -1285,22 +1297,21 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)
return;
/* remove all entries with from=0 */
- lbr_nr = cpu_lbr_stack(cpuc)->nr;
+ lbr_nr = lbr_stack->nr;
for (i = 0; i < lbr_nr; ) {
- if (!cpu_lbr_stack(cpuc)->entries[i].from) {
+ if (!lbr_stack->entries[i].from) {
j = i;
while (++j < lbr_nr) {
- cpu_lbr_stack(cpuc)->entries[j-1] =
- cpu_lbr_stack(cpuc)->entries[j];
- cpuc->lbr_counters[j-1] = cpuc->lbr_counters[j];
+ lbr_stack->entries[j-1] = lbr_stack->entries[j];
+ lbr_counters[j-1] = lbr_counters[j];
}
lbr_nr--;
- if (!cpu_lbr_stack(cpuc)->entries[i].from)
+ if (!lbr_stack->entries[i].from)
continue;
}
i++;
}
- cpu_lbr_stack(cpuc)->nr = lbr_nr;
+ lbr_stack->nr = lbr_nr;
}
void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr)
@@ -1314,8 +1325,8 @@ void intel_pmu_store_pebs_lbrs(struct lbr_entry *lbr)
else
cpu_lbr_stack(cpuc)->hw_idx = intel_pmu_lbr_tos();
- intel_pmu_store_lbr(cpuc, lbr);
- intel_pmu_lbr_filter(cpuc);
+ intel_pmu_store_lbr(lbr, cpu_lbr_stack(cpuc));
+ intel_pmu_lbr_filter(cpuc, cpu_lbr_stack(cpuc), cpuc->lbr_counters);
}
/*
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index e3d625a6d7eb..fa0af528cd1e 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -594,13 +594,13 @@ static __always_inline struct perf_branch_stack *cpu_lbr_stack(struct cpu_hw_eve
return (struct perf_branch_stack *)cpuc->__lbr_stack;
}
-static __always_inline void cpu_init_lbr_stack(struct cpu_hw_events *cpuc, int nr)
+static __always_inline void cpu_init_lbr_stack(struct perf_branch_stack *lbr_stack, int nr)
{
/*
* Initialize the branch-stack count before populating entries[]
* to avoid UBSAN complaints from the __counted_by(nr) flexible array.
*/
- cpu_lbr_stack(cpuc)->nr = nr;
+ lbr_stack->nr = nr;
}
#define __EVENT_CONSTRAINT_RANGE(c, e, n, m, w, o, f) { \
@@ -1207,7 +1207,8 @@ struct x86_pmu {
unsigned int lbr_counters:4;
void (*lbr_reset)(void);
- void (*lbr_read)(struct cpu_hw_events *cpuc);
+ void (*lbr_read)(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack);
void (*lbr_save)(void *ctx);
void (*lbr_restore)(void *ctx);
@@ -2007,11 +2008,14 @@ void intel_pmu_lbr_enable_all(bool pmi);
void intel_pmu_lbr_disable_all(void);
-void intel_pmu_lbr_read(void);
+void intel_pmu_lbr_read(struct perf_branch_stack *lbr_stack,
+ u64 *lbr_counters);
-void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc);
+void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack);
-void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc);
+void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc,
+ struct perf_branch_stack *lbr_stack);
void intel_pmu_lbr_save(void *ctx);
--
2.34.1
next prev parent reply other threads:[~2026-11-06 4:41 UTC|newest]
Thread overview: 46+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-06 4:32 [Patch v6 00/20] perf/x86: Fix multiple LBR sampling issues Dapeng Mi
2026-11-06 4:32 ` [Patch v6 01/20] perf/x86: Disable LBR if x86_pmu.lbr_nr exceeds MAX_LBR_ENTRIES Dapeng Mi
2026-11-06 5:01 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 02/20] perf/x86/intel: Validate LBR XSAVE state before updating LBR stack Dapeng Mi
2026-11-06 5:05 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 03/20] perf/x86/amd/lbr: Limit branch snapshots to valid entries Dapeng Mi
2026-11-06 4:50 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 04/20] perf/x86/intel/lbr: " Dapeng Mi
2026-11-06 4:52 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 05/20] perf/x86: Add lbr_enabled to track LBR state Dapeng Mi
2026-11-06 5:10 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 06/20] perf/x86/amd: Avoid re-enabling PMU and LBR for BPF when disabled Dapeng Mi
2026-11-06 5:01 ` sashiko-bot
2026-11-06 7:07 ` Mi, Dapeng
2026-11-06 4:32 ` [Patch v6 07/20] perf/x86/intel: Add helper to enable PMU without LBR Dapeng Mi
2026-11-06 4:45 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 08/20] perf/x86/intel: Avoid re-enabling PMU and LBR for BPF when disabled Dapeng Mi
2026-11-06 5:06 ` sashiko-bot
2026-11-06 7:09 ` Mi, Dapeng
2026-11-06 4:32 ` [Patch v6 09/20] perf/x86/intel: Keep LBR frozen throughout PMI handling Dapeng Mi
2026-11-06 4:58 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 10/20] perf: Add counted-bounds tracking for perf_branch_stack Dapeng Mi
2026-11-06 5:00 ` sashiko-bot
2026-11-06 4:32 ` Dapeng Mi [this message]
2026-11-06 5:09 ` [Patch v6 11/20] perf/x86/intel: Allow LBR read helpers to operate on explicit stacks sashiko-bot
2026-11-06 4:32 ` [Patch v6 12/20] perf/x86/intel: Use a dedicated LBR stack for BPF snapshotting Dapeng Mi
2026-11-06 5:17 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 13/20] perf/x86/intel: Always snapshot live hardware LBR state for BPF Dapeng Mi
2026-11-06 5:13 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 14/20] perf/x86/amd: Refresh LBR stack in PMI only when LBR users are active Dapeng Mi
2026-11-06 5:09 ` sashiko-bot
2026-11-12 9:59 ` Mi, Dapeng
2026-11-06 4:32 ` [Patch v6 15/20] perf/x86/intel: Refresh LBR stack for non-PEBS LBR events when needed Dapeng Mi
2026-11-06 5:58 ` sashiko-bot
2026-11-06 6:54 ` Mi, Dapeng
2026-11-06 4:32 ` [Patch v6 16/20] perf/x86/intel: Remove dead lbr_pebs_users Dapeng Mi
2026-11-06 5:08 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 17/20] perf/x86/intel: Clear raw LBR counters in the LBR stack Dapeng Mi
2026-11-06 5:17 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 18/20] perf/x86/intel: Refill LBR stack for LBR counter events in PMI handler Dapeng Mi
2026-11-06 5:17 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 19/20] perf/x86/intel: Pass LBR stack size into store_lbr() Dapeng Mi
2026-11-06 5:12 ` sashiko-bot
2026-11-06 4:32 ` [Patch v6 20/20] perf/core: Don't overwrite shared LBR stack depth Dapeng Mi
2026-11-06 5:17 ` sashiko-bot
2026-11-06 7:23 ` [Patch v6 00/20] perf/x86: Fix multiple LBR sampling issues Mi, Dapeng
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261106043257.842864-12-dapeng1.mi@linux.intel.com \
--to=dapeng1.mi@linux.intel.com \
--cc=acme@kernel.org \
--cc=adrian.hunter@intel.com \
--cc=ak@linux.intel.com \
--cc=alexander.shishkin@linux.intel.com \
--cc=ananth.narayan@amd.com \
--cc=andrii@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=dapeng1.mi@intel.com \
--cc=eranian@google.com \
--cc=irogers@google.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-perf-users@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=namhyung@kernel.org \
--cc=peterz@infradead.org \
--cc=ravi.bangoria@amd.com \
--cc=sandipan.das@amd.com \
--cc=song@kernel.org \
--cc=thomas.falcon@intel.com \
--cc=xudong.hao@intel.com \
--cc=zide.chen@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.