[PATCH v4 28/31] x86,fs/resctrl: Fix RMID allocation for multiple monitor resources

[Tony Luck <tony.luck@intel.com>]

The resctrl file system code assumed that the only monitor events were
tied to the RDT_RESOURCE_L3 resource. Also that the number of supported
RMIDs was enumerated during early initialization.

RDT_RESOURCE_PERF_PKG breaks both of those assumptions.

Delay the final enumeration of the number of RMIDs and subsequent
allocation of structures until first mount of the resctrl file system.

Signed-off-by: Tony Luck <tony.luck@intel.com>
---
 fs/resctrl/internal.h              |  4 +--
 arch/x86/kernel/cpu/resctrl/core.c |  8 +++--
 fs/resctrl/monitor.c               | 48 +++++++++++++-----------------
 fs/resctrl/rdtgroup.c              | 13 ++++----
 4 files changed, 35 insertions(+), 38 deletions(-)

diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h
index 623a9fadc18a..fb5ae8ba0c17 100644
--- a/fs/resctrl/internal.h
+++ b/fs/resctrl/internal.h
@@ -364,7 +364,7 @@ int alloc_rmid(u32 closid);
 
 void free_rmid(u32 closid, u32 rmid);
 
-void resctrl_mon_resource_exit(void);
+void resctrl_dom_data_exit(void);
 
 void mon_event_count(void *info);
 
@@ -405,7 +405,7 @@ enum resctrl_event_id resctrl_get_mon_event_by_name(char *name);
 
 char *resctrl_mon_event_name(enum resctrl_event_id evt);
 
-void resctrl_init_mon_events(void);
+int resctrl_init_mon_events(void);
 
 #ifdef CONFIG_RESCTRL_FS_PSEUDO_LOCK
 int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp);
diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
index dc312e24ab87..d921f32a1b6c 100644
--- a/arch/x86/kernel/cpu/resctrl/core.c
+++ b/arch/x86/kernel/cpu/resctrl/core.c
@@ -112,10 +112,14 @@ struct rdt_hw_resource rdt_resources_all[RDT_NUM_RESOURCES] = {
 
 u32 resctrl_arch_system_num_rmid_idx(void)
 {
-	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
+	struct rdt_resource *r;
+	int num_rmids = S32_MAX;
+
+	for_each_mon_capable_rdt_resource(r)
+		num_rmids = min(num_rmids, r->num_rmid);
 
 	/* RMID are independent numbers for x86. num_rmid_idx == num_rmid */
-	return r->num_rmid;
+	return num_rmids;
 }
 
 struct rdt_resource *resctrl_arch_get_resource(enum resctrl_res_level l)
diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c
index f848325591b4..f7a5ffe9be25 100644
--- a/fs/resctrl/monitor.c
+++ b/fs/resctrl/monitor.c
@@ -762,7 +762,7 @@ void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, unsigned long del
 		schedule_delayed_work_on(cpu, &dom->mbm_over, delay);
 }
 
-static int dom_data_init(struct rdt_resource *r)
+static int resctrl_dom_data_init(struct rdt_resource *r)
 {
 	u32 idx_limit = resctrl_arch_system_num_rmid_idx();
 	u32 num_closid = resctrl_arch_get_num_closid(r);
@@ -770,7 +770,10 @@ static int dom_data_init(struct rdt_resource *r)
 	int err = 0, i;
 	u32 idx;
 
-	mutex_lock(&rdtgroup_mutex);
+	/* Are there any mon_capable resources? */
+	if (idx_limit == S32_MAX)
+		return 0;
+
 	if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) {
 		u32 *tmp;
 
@@ -783,7 +786,7 @@ static int dom_data_init(struct rdt_resource *r)
 		tmp = kcalloc(num_closid, sizeof(*tmp), GFP_KERNEL);
 		if (!tmp) {
 			err = -ENOMEM;
-			goto out_unlock;
+			goto out;
 		}
 
 		closid_num_dirty_rmid = tmp;
@@ -796,7 +799,7 @@ static int dom_data_init(struct rdt_resource *r)
 			closid_num_dirty_rmid = NULL;
 		}
 		err = -ENOMEM;
-		goto out_unlock;
+		goto out;
 	}
 
 	for (i = 0; i < idx_limit; i++) {
@@ -817,14 +820,15 @@ static int dom_data_init(struct rdt_resource *r)
 	entry = __rmid_entry(idx);
 	list_del(&entry->list);
 
-out_unlock:
-	mutex_unlock(&rdtgroup_mutex);
+out:
 
 	return err;
 }
 
-static void dom_data_exit(struct rdt_resource *r)
+void resctrl_dom_data_exit(void)
 {
+	struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
+
 	mutex_lock(&rdtgroup_mutex);
 
 	if (!r->mon_capable)
@@ -954,16 +958,21 @@ char *resctrl_mon_event_name(enum resctrl_event_id evt)
  * events have been enumerated. Only needs to build the per-resource
  * event lists once.
  */
-void resctrl_init_mon_events(void)
+int resctrl_init_mon_events(void)
 {
+	struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
 	enum resctrl_event_id evt;
-	struct rdt_resource *r;
 	static bool only_once;
+	int ret;
 
 	if (only_once)
-		return;
+		return 0;
 	only_once = true;
 
+	ret = resctrl_dom_data_init(r);
+	if (ret)
+		return ret;
+
 	for_each_mon_capable_rdt_resource(r)
 		INIT_LIST_HEAD(&r->evt_list);
 
@@ -973,6 +982,8 @@ void resctrl_init_mon_events(void)
 		r = resctrl_arch_get_resource(mon_event_all[evt].rid);
 		list_add_tail(&mon_event_all[evt].list, &r->evt_list);
 	}
+
+	return ret;
 }
 
 /**
@@ -989,16 +1000,6 @@ void resctrl_init_mon_events(void)
  */
 int resctrl_mon_resource_init(void)
 {
-	struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
-	int ret;
-
-	if (!r->mon_capable)
-		return 0;
-
-	ret = dom_data_init(r);
-	if (ret)
-		return ret;
-
 	if (resctrl_arch_is_evt_configurable(QOS_L3_MBM_TOTAL_EVENT_ID)) {
 		mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID].configurable = true;
 		resctrl_file_fflags_init("mbm_total_bytes_config",
@@ -1017,10 +1018,3 @@ int resctrl_mon_resource_init(void)
 
 	return 0;
 }
-
-void resctrl_mon_resource_exit(void)
-{
-	struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3);
-
-	dom_data_exit(r);
-}
diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c
index 544fa721e067..195e41eb73fb 100644
--- a/fs/resctrl/rdtgroup.c
+++ b/fs/resctrl/rdtgroup.c
@@ -2595,7 +2595,9 @@ static int rdt_get_tree(struct fs_context *fc)
 		goto out;
 	}
 
-	resctrl_init_mon_events();
+	ret = resctrl_init_mon_events();
+	if (ret)
+		goto out;
 
 	ret = rdtgroup_setup_root(ctx);
 	if (ret)
@@ -4300,10 +4302,8 @@ int resctrl_init(void)
 		return ret;
 
 	ret = sysfs_create_mount_point(fs_kobj, "resctrl");
-	if (ret) {
-		resctrl_mon_resource_exit();
+	if (ret)
 		return ret;
-	}
 
 	ret = register_filesystem(&rdt_fs_type);
 	if (ret)
@@ -4336,7 +4336,6 @@ int resctrl_init(void)
 
 cleanup_mountpoint:
 	sysfs_remove_mount_point(fs_kobj, "resctrl");
-	resctrl_mon_resource_exit();
 
 	return ret;
 }
@@ -4363,7 +4362,7 @@ static bool resctrl_online_domains_exist(void)
  * When called by the architecture code, all CPUs and resctrl domains must be
  * offline. This ensures the limbo and overflow handlers are not scheduled to
  * run, meaning the data structures they access can be freed by
- * resctrl_mon_resource_exit().
+ * resctrl_dom_data_exit().
  *
  * After this function has returned, the architecture code should return an
  * from all resctrl_arch_ functions that can do this.
@@ -4390,5 +4389,5 @@ void resctrl_exit(void)
 	 * it can be used to umount resctrl.
 	 */
 
-	resctrl_mon_resource_exit();
+	resctrl_dom_data_exit();
 }
-- 
2.48.1