[PATCH 3/5] selftests/resctrl: Support the MPAM memory bandwidth monitoring layout

[Richard Cheng <icheng@nvidia.com>]

Arm MPAM systems running resctrl expose memory bandwidth differently
from x86:

 - MB control/monitor domains are keyed by NUMA node id, which need
   not match the L3 cache id (on a Grace/Vera system L3 ids are {1,2}
   while MB domains are {0,1,2,10,...}).
 - Monitoring lives in a separate MB_MON resource that provides only
   mbm_total_bytes; there is no local/total split and no L3_MON
   bandwidth event.
 - In "mbm_event" (ABMC) assignment mode a hardware counter must be
   explicitly assigned to a group+domain+event before the bandwidth
   file returns a value (it reads "Unassigned" otherwise), and
   removing a group does not release its counter, so it must be
   unassigned explicitly or it leaks.

Teach the test framework this layout:

 - get_domain_id() resolves MB domains via the CPU's NUMA node when
   the MB_MON resource exists. x86 (no MB_MON) is unaffected.
 - initialize_mem_bw_resctrl() falls back to
   mon_MB_<domain>/mbm_total_bytes when mbm_local_bytes is absent,
   and assigns an ABMC counter when the group's mbm_MB_assignments
   file is present. Note the assignment parser requires a trailing
   newline; without it the write fails with EINVAL.
 - mem_bw_ref_cleanup() unassigns the counter again.
 - The MBA feature check accepts MB_MON's mbm_total_bytes as the
   monitoring source.

Comparing a reference PMU total (read+write) with mbm_total_bytes is
a like-for-like comparison; the iMC path keeps comparing read
bandwidth with mbm_local_bytes as before.

No effect yet on any vendor: the MBA test is still gated to Intel.

Signed-off-by: Richard Cheng <icheng@nvidia.com>
---
 tools/testing/selftests/resctrl/mba_test.c    |  3 +-
 tools/testing/selftests/resctrl/resctrl_val.c | 69 +++++++++++++++++--
 tools/testing/selftests/resctrl/resctrlfs.c   | 45 +++++++++++-
 3 files changed, 111 insertions(+), 6 deletions(-)

diff --git a/tools/testing/selftests/resctrl/mba_test.c b/tools/testing/selftests/resctrl/mba_test.c
index 8b891d3e6650..6c85107c32ee 100644
--- a/tools/testing/selftests/resctrl/mba_test.c
+++ b/tools/testing/selftests/resctrl/mba_test.c
@@ -211,7 +211,8 @@ static int mba_run_test(const struct resctrl_test *test, const struct user_param
 static bool mba_feature_check(const struct resctrl_test *test)
 {
 	return test_resource_feature_check(test) &&
-	       resctrl_mon_feature_exists("L3_MON", "mbm_local_bytes") &&
+	       (resctrl_mon_feature_exists("L3_MON", "mbm_local_bytes") ||
+		resctrl_mon_feature_exists("MB_MON", "mbm_total_bytes")) &&
 	       mem_bw_ref_available();
 }
 
diff --git a/tools/testing/selftests/resctrl/resctrl_val.c b/tools/testing/selftests/resctrl/resctrl_val.c
index a2b919bd4839..0c3079d31340 100644
--- a/tools/testing/selftests/resctrl/resctrl_val.c
+++ b/tools/testing/selftests/resctrl/resctrl_val.c
@@ -20,6 +20,21 @@
 #define CON_MBM_LOCAL_BYTES_PATH		\
 	"%s/%s/mon_data/mon_L3_%02d/mbm_local_bytes"
 
+/*
+ * MPAM exposes only total (read+write) bandwidth, under MB_MON, and per the
+ * NUMA-node-keyed MB domain rather than the L3 cache id.
+ */
+#define CON_MBM_TOTAL_BYTES_PATH		\
+	"%s/%s/mon_data/mon_MB_%02d/mbm_total_bytes"
+
+/*
+ * In MPAM "mbm_event" (ABMC) counter-assignment mode a hardware counter must
+ * be explicitly assigned to a group+domain+event before the bandwidth file
+ * reads a value; otherwise it returns "Unassigned".
+ */
+#define MBM_MB_ASSIGN_PATH			\
+	"%s/%s/mbm_MB_assignments"
+
 struct membw_read_format {
 	__u64 value;         /* The value of the event */
 	__u64 time_enabled;  /* if PERF_FORMAT_TOTAL_TIME_ENABLED */
@@ -58,6 +73,10 @@ static struct mem_bw_counter bw_counters[MAX_BW_COUNTERS];
 static const struct resctrl_test *current_test;
 static const struct mem_bw_backend *mem_bw_backend;
 
+/* MPAM ABMC counter assignment state, see initialize_mem_bw_resctrl(). */
+static char mbm_assign_path[1024];
+static int mbm_assign_domain = -1;
+
 static void read_mem_bw_initialize_perf_event_attr(int i)
 {
 	memset(&bw_counters[i].pe, 0,
@@ -452,21 +471,63 @@ static int get_mem_bw_ref(float *bw_ref)
  * initialize_mem_bw_resctrl:	Appropriately populate "mbm_total_path"
  * @param:	Parameters passed to resctrl_val()
  * @domain_id:	Domain ID (cache ID; for MB, L3 cache ID)
+ *
+ * x86 resctrl reports per-RMID local bandwidth under L3_MON. MPAM exposes only
+ * total (read+write) bandwidth under MB_MON, indexed by the NUMA-node-keyed MB
+ * domain, and in "mbm_event" (ABMC) mode a hardware counter must be assigned to
+ * the group+domain before the file reads a value.
  */
 void initialize_mem_bw_resctrl(const struct resctrl_val_param *param,
 			       int domain_id)
 {
-	sprintf(mbm_total_path, CON_MBM_LOCAL_BYTES_PATH, RESCTRL_PATH,
-		param->ctrlgrp, domain_id);
+	const char *grp = param->ctrlgrp ? param->ctrlgrp : "";
+
+	if (resctrl_mon_feature_exists("L3_MON", "mbm_local_bytes")) {
+		sprintf(mbm_total_path, CON_MBM_LOCAL_BYTES_PATH, RESCTRL_PATH,
+			grp, domain_id);
+		return;
+	}
+
+	/* MPAM: total bandwidth under MB_MON. */
+	sprintf(mbm_total_path, CON_MBM_TOTAL_BYTES_PATH, RESCTRL_PATH,
+		grp, domain_id);
+
+	/* Assign an ABMC counter for this group+domain if in mbm_event mode. */
+	snprintf(mbm_assign_path, sizeof(mbm_assign_path), MBM_MB_ASSIGN_PATH,
+		 RESCTRL_PATH, grp);
+	if (access(mbm_assign_path, W_OK) == 0) {
+		FILE *fp = fopen(mbm_assign_path, "w");
+
+		if (fp) {
+			/* The assignment parser requires a trailing newline. */
+			fprintf(fp, "mbm_total_bytes:%d=e\n", domain_id);
+			fclose(fp);
+			mbm_assign_domain = domain_id;
+		}
+	} else {
+		/* Not in counter-assignment mode (or no MB_MON); nothing to do. */
+		mbm_assign_path[0] = '\0';
+	}
 }
 
 /*
- * mem_bw_ref_cleanup - Reset the selected reference-bandwidth backend
+ * mem_bw_ref_cleanup - Release an assigned ABMC counter and reset backend state
  *
- * Safe to call when no backend was selected.
+ * Group removal does not reclaim assigned MPAM counters, so the test must
+ * unassign explicitly. Safe to call when nothing was assigned.
  */
 void mem_bw_ref_cleanup(void)
 {
+	if (mbm_assign_domain >= 0 && mbm_assign_path[0]) {
+		FILE *fp = fopen(mbm_assign_path, "w");
+
+		if (fp) {
+			fprintf(fp, "mbm_total_bytes:%d=_\n", mbm_assign_domain);
+			fclose(fp);
+		}
+	}
+	mbm_assign_domain = -1;
+	mbm_assign_path[0] = '\0';
 	mem_bw_backend = NULL;
 }
 
diff --git a/tools/testing/selftests/resctrl/resctrlfs.c b/tools/testing/selftests/resctrl/resctrlfs.c
index b9c1bfb6cc02..893f6748c68c 100644
--- a/tools/testing/selftests/resctrl/resctrlfs.c
+++ b/tools/testing/selftests/resctrl/resctrlfs.c
@@ -121,11 +121,40 @@ static int get_resource_cache_level(const char *resource)
 	return get_cache_level(resource);
 }
 
+/*
+ * cpu_to_numa_node - NUMA node a logical CPU belongs to
+ * @cpu_no:	CPU number
+ *
+ * Reads the "node<N>" symlink in the CPU's sysfs directory.
+ *
+ * Return: NUMA node id (>= 0) on success, < 0 on failure.
+ */
+static int cpu_to_numa_node(int cpu_no)
+{
+	char cpu_dir[1024];
+	struct dirent *ep;
+	int node = -1;
+	DIR *dp;
+
+	snprintf(cpu_dir, sizeof(cpu_dir), "%s%d", PHYS_ID_PATH, cpu_no);
+	dp = opendir(cpu_dir);
+	if (!dp)
+		return -1;
+	while ((ep = readdir(dp))) {
+		if (!strncmp(ep->d_name, "node", 4) && isdigit(ep->d_name[4])) {
+			node = atoi(ep->d_name + 4);
+			break;
+		}
+	}
+	closedir(dp);
+	return node;
+}
+
 /*
  * get_domain_id - Get resctrl domain ID for a specified CPU
  * @resource:	resource name
  * @cpu_no:	CPU number
- * @domain_id:	domain ID (cache ID; for MB, L3 cache ID)
+ * @domain_id:	domain ID (cache ID; for MB, L3 cache ID, or NUMA node on MPAM)
  *
  * Return: >= 0 on success, < 0 on failure.
  */
@@ -135,6 +164,20 @@ int get_domain_id(const char *resource, int cpu_no, int *domain_id)
 	int cache_num;
 	FILE *fp;
 
+	/*
+	 * On MPAM the MB monitoring/allocation domains are keyed by NUMA node
+	 * (surfaced via a separate MB_MON group) and need not match the L3 cache
+	 * id used on x86, so resolve the MB domain from the CPU's NUMA node.
+	 */
+	if (!strcmp(resource, "MB") && resctrl_resource_exists("MB_MON")) {
+		int node = cpu_to_numa_node(cpu_no);
+
+		if (node >= 0) {
+			*domain_id = node;
+			return 0;
+		}
+	}
+
 	cache_num = get_resource_cache_level(resource);
 	if (cache_num < 0)
 		return cache_num;
-- 
2.43.0