[PATCH v3 16/46] perf/x86/intel/cmt: set sched rmid and complete pmu start/stop/add/del

[David Carrillo-Cisneros <davidcc@google.com>]

Now that the pmonr state machine and pqr_common are in place, add
pmonr_update_sched_rmid to find the adequate rmid to use. With it,
complete the body of PMUs functions that start/stop, add/del events.

A pmonr in Unused state tries to allocate a free rmid the first time one
of its monitored threads is scheduled in a CPU package (lazy allocation
of rmids). If there is no available rmids in that package, the pmonr
enters a Dep_Idle state (it borrows the sched_rmid from its
Lowest Monitored Ancestor (lma) pmonr).

When a event is stopped, no other event runs in that CPU and PQR msr
uses the rmid of the monr_hrchy_root's pmonr for that CPU's package.

Details in pmonr state machine's comments.

Signed-off-by: David Carrillo-Cisneros <davidcc@google.com>
---
 arch/x86/events/intel/cmt.c | 101 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 101 insertions(+)

diff --git a/arch/x86/events/intel/cmt.c b/arch/x86/events/intel/cmt.c
index ce5be74..9421a3e 100644
--- a/arch/x86/events/intel/cmt.c
+++ b/arch/x86/events/intel/cmt.c
@@ -650,6 +650,74 @@ static int monr_append_event(struct monr *monr, struct perf_event *event)
 	return err;
 }
 
+/**
+ * pmonr_update_sched_rmid() - Update sched_rmid for @pmonr in current package.
+ *
+ * Always finds valid rmids for non Off pmonr. Safe to call with IRQs disabled.
+ * A lock-free fast path reuses the rmid when the pmonr has been scheduled
+ * before in this package. Otherwise, tries to get a free rmid. On failure,
+ * enters Dep_Idle state and uses the rmid of its lender. There is always a
+ * pmonr to borrow from since monr_hrchy_root has all its pmonrs in Active
+ * state.
+ * Return: new pmonr_rmids for pmonr.
+ */
+static inline union pmonr_rmids pmonr_update_sched_rmid(struct pmonr *pmonr)
+{
+	struct pkg_data *pkgd = pmonr->pkgd;
+	union pmonr_rmids rmids;
+	u32 free_rmid;
+
+	/* Use atomic_rmids to check state in a lock-free fastpath. */
+	rmids.value = atomic64_read(&pmonr->atomic_rmids);
+	if (rmids.sched_rmid != INVALID_RMID)
+		return rmids;
+
+	/* No need to obtain RMID if in Off state. */
+	if (rmids.sched_rmid == rmids.read_rmid)
+		return rmids;
+
+	/*
+	 * Lock-free path failed. Now acquire lock and verify that state
+	 * and atomic_rmids haven't changed. If still Unused, try to
+	 * obtain a free RMID.
+	 */
+	raw_spin_lock(&pkgd->lock);
+
+	/* With lock acquired it is ok to read pmonr::state. */
+	if (pmonr->state != PMONR_UNUSED) {
+		/* Update rmids in case they changed before acquiring lock. */
+		rmids.value = atomic64_read(&pmonr->atomic_rmids);
+		raw_spin_unlock(&pkgd->lock);
+		return rmids;
+	}
+
+	free_rmid = find_first_bit(pkgd->free_rmids, CMT_MAX_NR_RMIDS);
+	if (free_rmid == CMT_MAX_NR_RMIDS)
+		pmonr_unused_to_dep_idle(pmonr);
+	else
+		pmonr_unused_to_active(pmonr, free_rmid);
+
+	raw_spin_unlock(&pkgd->lock);
+
+	rmids.value = atomic64_read(&pmonr->atomic_rmids);
+
+	return rmids;
+}
+
+static inline union pmonr_rmids monr_get_sched_in_rmids(struct monr *monr)
+{
+	struct pmonr *pmonr;
+	union pmonr_rmids rmids;
+	u16 pkgid = topology_logical_package_id(smp_processor_id());
+
+	rcu_read_lock();
+	pmonr = rcu_dereference(monr->pmonrs[pkgid]);
+	rmids = pmonr_update_sched_rmid(pmonr);
+	rcu_read_unlock();
+
+	return rmids;
+}
+
 static void monr_hrchy_insert_leaf(struct monr *monr, struct monr *parent)
 {
 	unsigned long flags;
@@ -865,16 +933,49 @@ static void intel_cmt_event_read(struct perf_event *event)
 {
 }
 
+static inline void __intel_cmt_event_start(struct perf_event *event,
+					   union pmonr_rmids rmids)
+{
+	if (!(event->hw.state & PERF_HES_STOPPED))
+		return;
+	event->hw.state &= ~PERF_HES_STOPPED;
+	pqr_cache_update_rmid(rmids.sched_rmid);
+}
+
 static void intel_cmt_event_start(struct perf_event *event, int mode)
 {
+	union pmonr_rmids rmids;
+
+	rmids = monr_get_sched_in_rmids(monr_from_event(event));
+	__intel_cmt_event_start(event, rmids);
 }
 
 static void intel_cmt_event_stop(struct perf_event *event, int mode)
 {
+	union pmonr_rmids rmids;
+
+	if (event->hw.state & PERF_HES_STOPPED)
+		return;
+	event->hw.state |= PERF_HES_STOPPED;
+	rmids = monr_get_sched_in_rmids(monr_hrchy_root);
+	/*
+	 * HW tracks the rmid even when event is not scheduled and event
+	 * reads occur even if event is Inactive. Therefore there is no need to
+	 * read when event is stopped.
+	 */
+	pqr_cache_update_rmid(rmids.sched_rmid);
 }
 
 static int intel_cmt_event_add(struct perf_event *event, int mode)
 {
+	union pmonr_rmids rmids;
+
+	event->hw.state = PERF_HES_STOPPED;
+	rmids = monr_get_sched_in_rmids(monr_from_event(event));
+
+	if (mode & PERF_EF_START)
+		__intel_cmt_event_start(event, rmids);
+
 	return 0;
 }
 
-- 
2.8.0.rc3.226.g39d4020