[PATCH 10/23] perf: Reschedule events across PMU partition transitions

[Zide Chen <zide.chen@intel.com>]

From: Kan Liang <kan.liang@linux.intel.com>

When entering a guest with PMU partitioning enabled, some counters
remain available to the host. Similar to the non-partitioning setup,
schedule out all events because a counter may become guest-owned.

Unlike the non-partitioned case, host events must then be rescheduled
so that !exclude_guest events can be scheduled onto the remaining
host-owned counters, while keeping exclude_guest events off. Since
PMU partitioning shrinks the counters available to host events, add a
lightweight PMU-context reschedule cpuctx_sched_in_all() instead of
rebuilding the entire CPU perf scheduling state.

When exiting the guest, schedule out host events so that they can be
rescheduled against the expanded set of host-owned counters. Similar to
guest entry, only PMU-level event placement needs to be rebuilt, so add
ctx_sched_out_all() instead of using the existing full CPU-context
scheduling API.

These context switches rely on the architectural PMU partition mask
being configured for the target context. On guest entry, the guest
PMU partition mask is expected to be active before events are
rescheduled. On guest exit, it is expected to be disabled so that
events are rescheduled with host counter constraints.

This differs from the perf core guest context, where guest_ctx_loaded
is updated at the end of the load/put guest context. As a result,
guest_ctx_loaded does not yet reflect the target context when the
rescheduling occurs.

Add pmu_partition_enabled to perf_{load,put}_guest_context(), and
hardcode to false temporarily until later patches.

Signed-off-by: Kan Liang <kan.liang@linux.intel.com>
Co-developed-by: Zide Chen <zide.chen@intel.com>
Signed-off-by: Zide Chen <zide.chen@intel.com>
---
 arch/x86/kvm/pmu.c         |  4 +--
 include/linux/perf_event.h |  4 +--
 kernel/events/core.c       | 52 +++++++++++++++++++++++++++++++++++---
 3 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 7f619a99a152..c022337d0bec 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -1390,7 +1390,7 @@ void kvm_mediated_pmu_load(struct kvm_vcpu *vcpu)
 
 	perf_pmu_partition_preload();
 
-	perf_load_guest_context();
+	perf_load_guest_context(false);
 
 	/*
 	 * Explicitly clear PERF_GLOBAL_CTRL, as "loading" the guest's context
@@ -1463,5 +1463,5 @@ void kvm_mediated_pmu_put(struct kvm_vcpu *vcpu)
 
 	perf_put_guest_lvtpc();
 
-	perf_put_guest_context();
+	perf_put_guest_context(false);
 }
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index bd952e09055d..9f66a4c49256 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1934,8 +1934,8 @@ extern u64 perf_event_pause(struct perf_event *event, bool reset);
 #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU
 int perf_create_mediated_pmu(u64 pmu_partition_mask);
 void perf_release_mediated_pmu(void);
-void perf_load_guest_context(void);
-void perf_put_guest_context(void);
+void perf_load_guest_context(bool pmu_partition_enabled);
+void perf_put_guest_context(bool pmu_partition_enabled);
 int arch_perf_set_pmu_partition_mask(u64 pmu_partition_mask);
 #endif
 
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 23375f8d2261..c9e7a2f0edc8 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -6442,8 +6442,25 @@ void perf_release_mediated_pmu(void)
 }
 EXPORT_SYMBOL_FOR_KVM(perf_release_mediated_pmu);
 
-/* When loading a guest's mediated PMU, schedule out all exclude_guest events. */
-void perf_load_guest_context(void)
+static void cpuctx_sched_in_all(struct perf_cpu_context *cpuctx,
+				enum event_type_t type)
+{
+	struct perf_event_pmu_context *pmu_ctx;
+
+	for_each_epc(pmu_ctx, &cpuctx->ctx, NULL, EVENT_GUEST)
+		__pmu_ctx_sched_in(pmu_ctx, type | EVENT_GUEST);
+
+	if (cpuctx->task_ctx) {
+		for_each_epc(pmu_ctx, cpuctx->task_ctx, NULL, EVENT_GUEST)
+			__pmu_ctx_sched_in(pmu_ctx, type | EVENT_GUEST);
+	}
+}
+
+/*
+ * When loading a guest's mediated PMU, schedule out all exclude_guest events.
+ * In PMU partitioning, reschedule host events onto host-owned counters.
+ */
+void perf_load_guest_context(bool pmu_partition_enabled)
 {
 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 
@@ -6461,6 +6478,12 @@ void perf_load_guest_context(void)
 		task_ctx_sched_out(cpuctx->task_ctx, NULL, EVENT_GUEST);
 	}
 
+	/* Reschedule !exclude_guest events onto host-owned counters. */
+	if (pmu_partition_enabled) {
+		cpuctx_sched_in_all(cpuctx, EVENT_PINNED);
+		cpuctx_sched_in_all(cpuctx, EVENT_FLEXIBLE);
+	}
+
 	perf_ctx_enable(&cpuctx->ctx, EVENT_GUEST);
 	if (cpuctx->task_ctx)
 		perf_ctx_enable(cpuctx->task_ctx, EVENT_GUEST);
@@ -6469,7 +6492,21 @@ void perf_load_guest_context(void)
 }
 EXPORT_SYMBOL_GPL(perf_load_guest_context);
 
-void perf_put_guest_context(void)
+static void ctx_sched_out_all(struct perf_event_context *ctx)
+{
+	struct perf_event_pmu_context *pmu_ctx;
+
+	if (!ctx)
+		return;
+
+	list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
+		if (perf_skip_pmu_ctx(pmu_ctx, EVENT_GUEST))
+			continue;
+		__pmu_ctx_sched_out(pmu_ctx, EVENT_ALL);
+	}
+}
+
+void perf_put_guest_context(bool pmu_partition_enabled)
 {
 	struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
 
@@ -6484,6 +6521,15 @@ void perf_put_guest_context(void)
 	if (cpuctx->task_ctx)
 		perf_ctx_disable(cpuctx->task_ctx, EVENT_GUEST);
 
+	if (pmu_partition_enabled) {
+		ctx_time_update(cpuctx, &cpuctx->ctx);
+		if (cpuctx->task_ctx)
+			ctx_time_update(cpuctx, cpuctx->task_ctx);
+
+		ctx_sched_out_all(&cpuctx->ctx);
+		ctx_sched_out_all(cpuctx->task_ctx);
+	}
+
 	perf_event_sched_in(cpuctx, cpuctx->task_ctx, NULL, EVENT_GUEST);
 
 	if (cpuctx->task_ctx)
-- 
2.55.0