Re: [PATCH RFC v2 12/19] arm_mpam: resctrl: Add NUMA node notifier for domain online/offline

[Ben Horgan <ben.horgan@arm.com>]

Hi Fenghua,

On 31/08/2026 18:22, Fenghua Yu wrote:
> From: James Morse <james.morse@arm.com>
> 
> To expose resctrl resources that contain CPU-less NUMA domains, resctrl
> needs to be told when a CPU-less NUMA domain comes online. This can't
> be done with the cpuhp callbacks.
> 
> Add a memory hotplug notifier, and use this to create and destroy
> resctrl domains.
> 
> Signed-off-by: James Morse <james.morse@arm.com>
> Signed-off-by: Fenghua Yu <fenghuay@nvidia.com>
> ---
>  drivers/resctrl/mpam_resctrl.c | 343 ++++++++++++++++++++++++++++++++-
>  include/linux/memory.h         |   1 +
>  2 files changed, 337 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c
> index cd9bebf5f0c2..8bb313efc716 100644
> --- a/drivers/resctrl/mpam_resctrl.c
> +++ b/drivers/resctrl/mpam_resctrl.c
> @@ -11,6 +11,8 @@
>  #include <linux/limits.h>
>  #include <linux/list.h>
>  #include <linux/math.h>
> +#include <linux/memory.h>
> +#include <linux/node.h>
>  #include <linux/printk.h>
>  #include <linux/rculist.h>
>  #include <linux/resctrl.h>
> @@ -1775,16 +1777,25 @@ void resctrl_arch_reset_all_ctrls(struct rdt_resource *r)
>  	mpam_reset_class_locked(res->class);
>  }
>  
> +static void mpam_resctrl_domain_hdr_init_mask(const struct cpumask *cpus,
> +					      int id_cpu,
> +					      struct mpam_component *comp,
> +					      enum resctrl_res_level rid,
> +					      struct rdt_domain_hdr *hdr)
> +{
> +	INIT_LIST_HEAD(&hdr->list);
> +	hdr->id = mpam_resctrl_pick_domain_id(id_cpu, comp);
> +	hdr->rid = rid;
> +	cpumask_copy(&hdr->cpu_mask, cpus);
> +}
> +
>  static void mpam_resctrl_domain_hdr_init(int cpu, struct mpam_component *comp,
>  					 enum resctrl_res_level rid,
>  					 struct rdt_domain_hdr *hdr)
>  {
>  	lockdep_assert_cpus_held();
>  
> -	INIT_LIST_HEAD(&hdr->list);
> -	hdr->id = mpam_resctrl_pick_domain_id(cpu, comp);
> -	hdr->rid = rid;
> -	cpumask_set_cpu(cpu, &hdr->cpu_mask);
> +	mpam_resctrl_domain_hdr_init_mask(cpumask_of(cpu), cpu, comp, rid, hdr);
>  }
>  
>  static void mpam_resctrl_online_domain_hdr(unsigned int cpu,
> @@ -1805,12 +1816,12 @@ static void mpam_resctrl_online_domain_hdr(unsigned int cpu,
>   * indicating the parent structure can be freed.
>   * If there are other CPUs in the domain, returns false.
>   */
> -static bool mpam_resctrl_offline_domain_hdr(unsigned int cpu,
> -					    struct rdt_domain_hdr *hdr)
> +static bool mpam_resctrl_offline_domain_hdr_mask(const struct cpumask *cpus,
> +						 struct rdt_domain_hdr *hdr)
>  {
>  	lockdep_assert_held(&domain_list_lock);
>  
> -	cpumask_clear_cpu(cpu, &hdr->cpu_mask);
> +	cpumask_andnot(&hdr->cpu_mask, &hdr->cpu_mask, cpus);
>  	if (cpumask_empty(&hdr->cpu_mask)) {
>  		list_del_rcu(&hdr->list);
>  		synchronize_rcu();
> @@ -1820,6 +1831,12 @@ static bool mpam_resctrl_offline_domain_hdr(unsigned int cpu,
>  	return false;
>  }
>  
> +static bool mpam_resctrl_offline_domain_hdr(unsigned int cpu,
> +					    struct rdt_domain_hdr *hdr)
> +{
> +	return mpam_resctrl_offline_domain_hdr_mask(cpumask_of(cpu), hdr);
> +}
> +
>  static void mpam_resctrl_domain_insert(struct list_head *list,
>  				       struct rdt_domain_hdr *new)
>  {
> @@ -1835,6 +1852,23 @@ static void mpam_resctrl_domain_insert(struct list_head *list,
>  	list_add_tail_rcu(&new->list, pos);
>  }
>  
> +static struct mpam_component *find_component_nid(struct mpam_class *class, int nid)
> +{
> +	struct mpam_component *comp;
> +
> +	if (!class || class->type != MPAM_CLASS_MEMORY)
> +		return NULL;
> +
> +	guard(srcu)(&mpam_srcu);
> +	list_for_each_entry_srcu(comp, &class->components, class_list,
> +				 srcu_read_lock_held(&mpam_srcu)) {
> +		if (comp->comp_id == nid)
> +			return comp;
> +	}
> +
> +	return NULL;
> +}
> +
>  static struct mpam_component *find_component(struct mpam_class *class, int cpu)
>  {
>  	struct mpam_component *comp;
> @@ -2094,6 +2128,297 @@ static void mpam_resctrl_offline_ctrls(unsigned int cpu, struct mpam_resctrl_res
>  	}
>  }
>  
> +static bool mpam_mba_uses_memory_nid(void)
> +{
> +	struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_MBA];
> +
> +	return res->class && res->class->type == MPAM_CLASS_MEMORY;
> +}
> +
> +static struct mpam_resctrl_dom *
> +mpam_get_ctrl_domain_from_nid(int nid, struct mpam_resctrl_res *res,
> +			      struct resctrl_ctrl *ctrl)
> +{
> +	struct mpam_resctrl_dom *dom;
> +
> +	list_for_each_entry(dom, &ctrl->domains, resctrl_ctrl_dom.hdr.list) {
> +		if (!dom->ctrl_comp || !dom->ctrl_comp->class)
> +			continue;
> +		if (dom->ctrl_comp->class->type != MPAM_CLASS_MEMORY)
> +			continue;
> +		if (dom->ctrl_comp->comp_id == nid)
> +			return dom;
> +	}
> +
> +	return NULL;
> +}
> +
> +static struct mpam_resctrl_dom *
> +mpam_get_mon_domain_from_nid(int nid, struct mpam_resctrl_res *res)
> +{
> +	struct mpam_resctrl_dom *dom;
> +	struct rdt_resource *r = &res->resctrl_res;
> +
> +	if (!r->mon_capable)
> +		return NULL;
> +
> +	list_for_each_entry(dom, &r->mon_domains, resctrl_mon_dom.hdr.list) {
> +		if (!dom->ctrl_comp || !dom->ctrl_comp->class)
> +			continue;
> +		if (dom->ctrl_comp->class->type != MPAM_CLASS_MEMORY)
> +			continue;
> +		if (dom->ctrl_comp->comp_id == nid)
> +			return dom;
> +	}
> +
> +	return NULL;
> +}
> +
> +static struct mpam_resctrl_dom *
> +mpam_resctrl_alloc_ctrl_domain_nid(int nid, struct mpam_resctrl_res *res,
> +				   struct resctrl_ctrl *ctrl,
> +				   struct mpam_component *comp)
> +{
> +	int err;
> +	struct mpam_resctrl_dom *dom;
> +	struct rdt_ctrl_domain *ctrl_d;
> +	struct rdt_resource *r = &res->resctrl_res;
> +	int id_cpu = cpumask_first(cpu_possible_mask);
> +
> +	if (id_cpu >= nr_cpu_ids)
> +		id_cpu = 0;
> +
> +	lockdep_assert_held(&domain_list_lock);
> +
> +	if (!r->alloc_capable)
> +		return ERR_PTR(-EINVAL);
> +
> +	if (WARN_ON_ONCE(!comp))
> +		return ERR_PTR(-EINVAL);
> +
> +	dom = kzalloc_node(sizeof(*dom), GFP_KERNEL, nid);
> +	if (!dom)
> +		return ERR_PTR(-ENOMEM);
> +
> +	dom->ctrl_comp = comp;
> +
> +	ctrl_d = &dom->resctrl_ctrl_dom;
> +	mpam_resctrl_domain_hdr_init_mask(cpu_possible_mask, id_cpu, comp,
> +					  r->rid, &ctrl_d->hdr);
> +	ctrl_d->hdr.type = RESCTRL_CTRL_DOMAIN;
> +	err = resctrl_online_ctrl_domain(r, ctrl, ctrl_d);
> +	if (err)
> +		goto free_domain;
> +
> +	mpam_resctrl_domain_insert(&ctrl->domains, &ctrl_d->hdr);
> +
> +	return dom;
> +
> +free_domain:
> +	kfree(dom);
> +	return ERR_PTR(err);
> +}
> +
> +static struct mpam_resctrl_dom *
> +mpam_resctrl_alloc_mon_domain_nid(int nid, struct mpam_resctrl_res *res,
> +				  struct mpam_component *comp)
> +{
> +	int err;
> +	struct mpam_resctrl_dom *dom;
> +	struct rdt_l3_mon_domain *mon_d;
> +	struct rdt_resource *r = &res->resctrl_res;
> +	struct mpam_component *any_mon_comp = NULL;
> +	struct mpam_resctrl_mon *mon;
> +	enum resctrl_event_id eventid;
> +	int id_cpu = cpumask_first(cpu_possible_mask);
> +
> +	if (id_cpu >= nr_cpu_ids)
> +		id_cpu = 0;
> +
> +	lockdep_assert_held(&domain_list_lock);
> +
> +	if (!r->mon_capable)
> +		return ERR_PTR(-EINVAL);
> +
> +	if (WARN_ON_ONCE(!comp))
> +		return ERR_PTR(-EINVAL);
> +
> +	dom = kzalloc_node(sizeof(*dom), GFP_KERNEL, nid);
> +	if (!dom)
> +		return ERR_PTR(-ENOMEM);
> +
> +	dom->ctrl_comp = comp;
> +
> +	for_each_mpam_resctrl_mon(mon, eventid) {
> +		struct mpam_component *mon_comp;
> +
> +		if (!mon->class)
> +			continue;
> +
> +		mon_comp = find_component_nid(mon->class, nid);
> +		dom->mon_comp[eventid] = mon_comp;
> +		if (mon_comp)
> +			any_mon_comp = mon_comp;
> +	}
> +	if (!any_mon_comp) {
> +		err = -EFAULT;
> +		goto free_domain;
> +	}
> +
> +	mon_d = &dom->resctrl_mon_dom;
> +	mpam_resctrl_domain_hdr_init_mask(cpu_possible_mask, id_cpu, any_mon_comp,
> +					  r->rid, &mon_d->hdr);
> +	mon_d->hdr.type = RESCTRL_MON_DOMAIN;
> +	err = resctrl_online_mon_domain(r, &mon_d->hdr);
> +	if (err)
> +		goto free_domain;
> +
> +	mpam_resctrl_domain_insert(&r->mon_domains, &mon_d->hdr);
> +
> +	return dom;
> +
> +free_domain:
> +	kfree(dom);
> +	return ERR_PTR(err);
> +}
> +
> +static int mpam_resctrl_online_node(unsigned int nid)
> +{
> +	struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_MBA];
> +	struct rdt_resource *r = &res->resctrl_res;
> +	struct mpam_component *comp;
> +	struct resctrl_ctrl *ctrl, *em_ctrl;
> +	struct mpam_resctrl_dom *dom;
> +
> +	if (!res->class)
> +		return 0;
> +
> +	comp = find_component_nid(res->class, nid);
> +	if (!comp)
> +		return 0;
> +
> +	guard(mutex)(&domain_list_lock);
> +
> +	if (r->alloc_capable) {
> +		for_each_resource_ctrl(ctrl, r) {
> +			dom = mpam_get_ctrl_domain_from_nid(nid, res, ctrl);
> +			if (!dom) {
> +				dom = mpam_resctrl_alloc_ctrl_domain_nid(nid, res,
> +									 ctrl, comp);
> +				if (IS_ERR(dom))
> +					return PTR_ERR(dom);
> +			}
> +
> +			list_for_each_entry(em_ctrl, &ctrl->emulated_by, entry) {
> +				dom = mpam_get_ctrl_domain_from_nid(nid, res, em_ctrl);
> +				if (!dom) {
> +					dom = mpam_resctrl_alloc_ctrl_domain_nid(nid, res,
> +										 em_ctrl, comp);
> +					if (IS_ERR(dom))
> +						return PTR_ERR(dom);
> +				}
> +			}
> +		}
> +	}
> +
> +	if (r->mon_capable) {
> +		dom = mpam_get_mon_domain_from_nid(nid, res);
> +		if (!dom) {
> +			dom = mpam_resctrl_alloc_mon_domain_nid(nid, res, comp);
> +			if (IS_ERR(dom))
> +				return PTR_ERR(dom);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
> +static int mpam_resctrl_offline_node(unsigned int nid)
> +{
> +	struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_MBA];
> +	struct rdt_resource *r = &res->resctrl_res;
> +	struct resctrl_ctrl *ctrl, *em_ctrl;
> +	struct mpam_resctrl_dom *dom;
> +	struct rdt_ctrl_domain *ctrl_d;
> +	struct rdt_l3_mon_domain *mon_d;
> +
> +	if (!res->class)
> +		return 0;
> +
> +	if (!find_component_nid(res->class, nid))
> +		return 0;
> +
> +	guard(mutex)(&domain_list_lock);
> +
> +	if (r->alloc_capable) {
> +		for_each_resource_ctrl(ctrl, r) {
> +			list_for_each_entry(em_ctrl, &ctrl->emulated_by, entry) {
> +				dom = mpam_get_ctrl_domain_from_nid(nid, res, em_ctrl);
> +				if (WARN_ON_ONCE(!dom))
> +					continue;
> +				ctrl_d = &dom->resctrl_ctrl_dom;
> +				if (mpam_resctrl_offline_domain_hdr_mask(cpu_possible_mask,
> +									 &ctrl_d->hdr)) {
> +					resctrl_offline_ctrl_domain(r, em_ctrl, ctrl_d);
> +					kfree(dom);
> +				}
> +			}
> +
> +			dom = mpam_get_ctrl_domain_from_nid(nid, res, ctrl);
> +			if (WARN_ON_ONCE(!dom))
> +				continue;
> +			ctrl_d = &dom->resctrl_ctrl_dom;
> +			if (mpam_resctrl_offline_domain_hdr_mask(cpu_possible_mask,
> +								 &ctrl_d->hdr)) {
> +				resctrl_offline_ctrl_domain(r, ctrl, ctrl_d);
> +				kfree(dom);
> +			}
> +		}
> +	}
> +
> +	if (r->mon_capable) {
> +		dom = mpam_get_mon_domain_from_nid(nid, res);
> +		if (WARN_ON_ONCE(!dom))
> +			return 0;
> +		mon_d = &dom->resctrl_mon_dom;
> +		if (mpam_resctrl_offline_domain_hdr_mask(cpu_possible_mask, &mon_d->hdr)) {
> +			resctrl_offline_mon_domain(r, &mon_d->hdr);
> +			kfree(dom);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
> +static int mpam_resctrl_node_notifier(struct notifier_block *self,
> +				      unsigned long action, void *arg)
> +{
> +	struct node_notify *nn = arg;
> +
> +	if (nn->nid < 0 || !mpam_mba_uses_memory_nid())
> +		return NOTIFY_OK;
> +
> +	/*
> +	 * Ignore nids that have CPUs. Resctrl needs to see the cpu offline
> +	 * call for each CPU to update the CPUs in control groups.
> +	 */
> +	if (!cpumask_empty(cpumask_of_node(nn->nid)))
> +		return NOTIFY_OK;

I was expecting the lifecycle of the resctrl MB_NODE resource to be the same as the memory
associated with the memory node. This is discussed on Reinette's controls PoC here:

https://lore.kernel.org/lkml/f5b6cec4-03d8-4a11-884d-d4579dab6b22@intel.com/

This looks to be a different lifecycle?

Thanks,

Ben

> +
> +	switch (action) {
> +	case NODE_ADDED_FIRST_MEMORY:
> +		mpam_resctrl_online_node(nn->nid);
> +		break;
> +	case NODE_REMOVED_LAST_MEMORY:
> +		mpam_resctrl_offline_node(nn->nid);
> +		break;
> +	default:
> +		break;
> +	}
> +
> +	return NOTIFY_OK;
> +}
> +
>  int mpam_resctrl_online_cpu(unsigned int cpu)
>  {
>  	struct mpam_resctrl_res *res;
> @@ -2229,6 +2554,10 @@ int mpam_resctrl_setup(void)
>  		}
>  	}
>  
> +	if (mpam_mba_uses_memory_nid())
> +		hotplug_node_notifier(mpam_resctrl_node_notifier,
> +				      RESCTRL_CALLBACK_PRI);
> +
>  	cpus_read_unlock();
>  
>  	if (!resctrl_arch_alloc_capable() && !resctrl_arch_mon_capable()) {
> diff --git a/include/linux/memory.h b/include/linux/memory.h
> index 463dc02f6cff..1497d4be38ee 100644
> --- a/include/linux/memory.h
> +++ b/include/linux/memory.h
> @@ -120,6 +120,7 @@ struct mem_section;
>  #define CPUSET_CALLBACK_PRI	10
>  #define MEMTIER_HOTPLUG_PRI	100
>  #define KSM_CALLBACK_PRI	100
> +#define RESCTRL_CALLBACK_PRI	100
>  
>  #ifndef CONFIG_MEMORY_HOTPLUG
>  static inline void memory_dev_init(void)