[PATCH v2 12/32] perf/x86/intel/cqm: schedule work for rotation task

[David Carrillo-Cisneros <davidcc@google.com>]

Make execution of rotation a delayed_work that does a best effort
to rotate __cqm_min_progress_rate pmonrs per-second in every package.

Reviewed-by: Stephane Eranian <eranian@google.com>
Signed-off-by: David Carrillo-Cisneros <davidcc@google.com>
---
 arch/x86/events/intel/cqm.c | 70 ++++++++++++++++++++++++++++++++++++++++++++-
 arch/x86/events/intel/cqm.h | 21 ++++++++++++++
 2 files changed, 90 insertions(+), 1 deletion(-)

diff --git a/arch/x86/events/intel/cqm.c b/arch/x86/events/intel/cqm.c
index 203fc66..a61dd70 100644
--- a/arch/x86/events/intel/cqm.c
+++ b/arch/x86/events/intel/cqm.c
@@ -222,6 +222,8 @@ static int pkg_data_init_cpu(int cpu)
 	mutex_init(&pkg_data->pkg_data_mutex);
 	raw_spin_lock_init(&pkg_data->pkg_data_lock);
 
+	INIT_DELAYED_WORK(
+		&pkg_data->rotation_work, intel_cqm_rmid_rotation_work);
 	/* XXX: Chose randomly*/
 	pkg_data->rotation_cpu = cpu;
 
@@ -1384,7 +1386,7 @@ read_nr_instate_pmonrs(struct pkg_data *pkg_data, u16 pkg_id) {
  * this point we can start reading values for the new RMID and treat the
  * old RMID as the free RMID for the next rotation.
  */
-void
+static void
 __intel_cqm_rmid_rotate(struct pkg_data *pkg_data,
 			unsigned int nr_max_limbo,
 			unsigned int nr_min_activated)
@@ -1519,6 +1521,70 @@ exit:
 
 static struct pmu intel_cqm_pmu;
 
+/* Rotation only needs to be run when there is any pmonr in (I)state. */
+static bool intel_cqm_need_rotation(u16 pkg_id)
+{
+
+	struct pkg_data *pkg_data;
+	bool need_rot;
+
+	pkg_data = cqm_pkgs_data[pkg_id];
+
+	mutex_lock_nested(&pkg_data->pkg_data_mutex, pkg_id);
+	/* Rotation is needed if prmids in limbo need to be recycled or if
+	 * there are pmonrs in (I)state.
+	 */
+	need_rot = !list_empty(&pkg_data->nopmonr_limbo_prmids_pool) ||
+		   !list_empty(&pkg_data->istate_pmonrs_lru);
+
+	mutex_unlock(&pkg_data->pkg_data_mutex);
+	return need_rot;
+}
+
+/*
+ * Schedule rotation in one package.
+ */
+static void __intel_cqm_schedule_rotation_for_pkg(u16 pkg_id)
+{
+	struct pkg_data *pkg_data;
+	unsigned long delay;
+
+	delay = msecs_to_jiffies(intel_cqm_pmu.hrtimer_interval_ms);
+	pkg_data = cqm_pkgs_data[pkg_id];
+	schedule_delayed_work_on(
+		pkg_data->rotation_cpu, &pkg_data->rotation_work, delay);
+}
+
+/*
+ * Schedule rotation and rmid's timed update in all packages.
+ * Reescheduling will stop when no longer needed.
+ */
+static void intel_cqm_schedule_work_all_pkgs(void)
+{
+	int pkg_id;
+
+	cqm_pkg_id_for_each_online(pkg_id)
+		__intel_cqm_schedule_rotation_for_pkg(pkg_id);
+}
+
+static void intel_cqm_rmid_rotation_work(struct work_struct *work)
+{
+	struct pkg_data *pkg_data = container_of(
+		to_delayed_work(work), struct pkg_data, rotation_work);
+	/* Allow max 25% of RMIDs to be in limbo. */
+	unsigned int max_limbo_rmids = max(1u, (pkg_data->max_rmid + 1) / 4);
+	unsigned int min_activated = max(1u, (intel_cqm_pmu.hrtimer_interval_ms
+		* __cqm_min_progress_rate) / 1000);
+	u16 pkg_id = topology_physical_package_id(pkg_data->rotation_cpu);
+
+	WARN_ON_ONCE(pkg_data != cqm_pkgs_data[pkg_id]);
+
+	__intel_cqm_rmid_rotate(pkg_data, max_limbo_rmids, min_activated);
+
+	if (intel_cqm_need_rotation(pkg_id))
+		__intel_cqm_schedule_rotation_for_pkg(pkg_id);
+}
+
 /*
  * Find a group and setup RMID.
  *
@@ -1744,6 +1810,8 @@ static int intel_cqm_event_init(struct perf_event *event)
 
 	mutex_unlock(&cqm_mutex);
 
+	intel_cqm_schedule_work_all_pkgs();
+
 	return 0;
 }
 
diff --git a/arch/x86/events/intel/cqm.h b/arch/x86/events/intel/cqm.h
index 12f4156..7e4e37a 100644
--- a/arch/x86/events/intel/cqm.h
+++ b/arch/x86/events/intel/cqm.h
@@ -239,6 +239,7 @@ struct pmonr {
  *				during process scheduling. The locks for all
  *				packages must be held when modifying the monr
  *				hierarchy.
+ * @rotation_work:		Task that performs rotation of prmids.
  * @rotation_cpu:               CPU to run @rotation_work on, it must be in the
  *                              package associated to this instance of pkg_data.
  */
@@ -268,6 +269,7 @@ struct pkg_data {
 	struct mutex		pkg_data_mutex;
 	raw_spinlock_t		pkg_data_lock;
 
+	struct delayed_work	rotation_work;
 	int			rotation_cpu;
 };
 
@@ -428,6 +430,18 @@ static inline int monr_hrchy_count_held_raw_spin_locks(void)
 #define CQM_DEFAULT_ROTATION_PERIOD 1200	/* ms */
 
 /*
+ * Rotation function.
+ * Rotation logic runs per-package. In each package, if free rmids are needed,
+ * it will steal prmids from the pmonr that has been the longest time in
+ * (A)state.
+ * The hardware provides to way to signal that a rmid will be reused, therefore,
+ * before reusing a rmid that has been stolen, the rmid should stay for some
+ * in a "limbo" state where is not associated to any thread, hoping that the
+ * cache lines allocated for this rmid will eventually be replaced.
+ */
+static void intel_cqm_rmid_rotation_work(struct work_struct *work);
+
+/*
  * Service Level Objectives (SLO) for the rotation logic.
  *
  * @__cqm_min_duration_mon_slice: Minimum duration of a monitored slice.
@@ -444,6 +458,13 @@ static unsigned int __cqm_min_mon_slice = CQM_DEFAULT_MIN_MON_SLICE;
 static unsigned int __cqm_max_wait_mon = CQM_DEFAULT_MAX_WAIT_MON;
 
 /*
+ * Minimum numbers of pmonrs that must go to Active state per second in order
+ * to consider rotation to be effective.
+ */
+#define CQM_DEFAULT_MIN_PROGRESS_RATE 1
+static unsigned int __cqm_min_progress_rate = CQM_DEFAULT_MIN_PROGRESS_RATE;
+
+/*
  * If we fail to assign any RMID for intel_cqm_rotation because cachelines are
  * still tagged with RMIDs in limbo even after having stolen enough rmids (a
  * maximum number of rmids in limbo at any time), then we increment the dirty
-- 
2.8.0.rc3.226.g39d4020