Re: [PATCH v6 08/42] x86/resctrl: Generate default_ctrl instead of sharing it

[Reinette Chatre <reinette.chatre@intel.com>]

Hi James,

On 2/7/25 10:17 AM, James Morse wrote:
> The struct rdt_resource default_ctrl is used by both the architecture
> code for resetting the hardware controls, and sometimes by the
> filesystem code as the default value for the schema, unless the
> bandwidth software controller is in use.
> 
> Having the default exposed by the architecture code causes unnecessary
> duplication for each architecture as the default value must be specified,
> but can be derived from other schema properties. Now that the
> maximum bandwidth is explicitly described, resctrl can derive the default
> value from the schema format and the other resource properties.
> 
> Signed-off-by: James Morse <james.morse@arm.com>
> Tested-by: Carl Worth <carl@os.amperecomputing.com> # arm64
> Tested-by: Shaopeng Tan <tan.shaopeng@jp.fujitsu.com>
> Reviewed-by: Shaopeng Tan <tan.shaopeng@jp.fujitsu.com>
> Reviewed-by: Tony Luck <tony.luck@intel.com>
> ---
> Changes since v5:
>  * Rewrote commit message.
> 
> Changes since v2:
>  * This patch is new.
> ---
>  arch/x86/kernel/cpu/resctrl/core.c        | 16 +++++++---------
>  arch/x86/kernel/cpu/resctrl/ctrlmondata.c |  5 +++--
>  arch/x86/kernel/cpu/resctrl/rdtgroup.c    |  6 +++---
>  include/linux/resctrl.h                   | 19 +++++++++++++++++--
>  4 files changed, 30 insertions(+), 16 deletions(-)
> 
> diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
> index 4504a12efc97..6fd195b600b1 100644
> --- a/arch/x86/kernel/cpu/resctrl/core.c
> +++ b/arch/x86/kernel/cpu/resctrl/core.c
> @@ -143,7 +143,10 @@ static inline void cache_alloc_hsw_probe(void)
>  {
>  	struct rdt_hw_resource *hw_res = &rdt_resources_all[RDT_RESOURCE_L3];
>  	struct rdt_resource *r  = &hw_res->r_resctrl;
> -	u64 max_cbm = BIT_ULL_MASK(20) - 1, l3_cbm_0;
> +	u64 max_cbm, l3_cbm_0;
> +
> +	r->cache.cbm_len = 20;
> +	max_cbm = resctrl_get_default_ctrl(r);
>  
>  	if (wrmsrl_safe(MSR_IA32_L3_CBM_BASE, max_cbm))
>  		return;
> @@ -155,8 +158,6 @@ static inline void cache_alloc_hsw_probe(void)
>  		return;
>  
>  	hw_res->num_closid = 4;
> -	r->default_ctrl = max_cbm;
> -	r->cache.cbm_len = 20;
>  	r->cache.shareable_bits = 0xc0000;
>  	r->cache.min_cbm_bits = 2;
>  	r->cache.arch_has_sparse_bitmasks = false;
> @@ -211,7 +212,6 @@ static __init bool __get_mem_config_intel(struct rdt_resource *r)
>  	cpuid_count(0x00000010, 3, &eax.full, &ebx, &ecx, &edx.full);
>  	hw_res->num_closid = edx.split.cos_max + 1;
>  	max_delay = eax.split.max_delay + 1;
> -	r->default_ctrl = MAX_MBA_BW;
>  	r->membw.max_bw = MAX_MBA_BW;
>  	r->membw.arch_needs_linear = true;
>  	if (ecx & MBA_IS_LINEAR) {
> @@ -250,7 +250,6 @@ static __init bool __rdt_get_mem_config_amd(struct rdt_resource *r)
>  
>  	cpuid_count(0x80000020, subleaf, &eax, &ebx, &ecx, &edx);
>  	hw_res->num_closid = edx + 1;
> -	r->default_ctrl = 1 << eax;
>  	r->membw.max_bw = 1 << eax;
>  
>  	/* AMD does not use delay */
> @@ -281,8 +280,7 @@ static void rdt_get_cache_alloc_cfg(int idx, struct rdt_resource *r)
>  	cpuid_count(0x00000010, idx, &eax.full, &ebx, &ecx.full, &edx.full);
>  	hw_res->num_closid = edx.split.cos_max + 1;
>  	r->cache.cbm_len = eax.split.cbm_len + 1;
> -	r->default_ctrl = BIT_MASK(eax.split.cbm_len + 1) - 1;
> -	r->cache.shareable_bits = ebx & r->default_ctrl;
> +	r->cache.shareable_bits = ebx & resctrl_get_default_ctrl(r);
>  	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
>  		r->cache.arch_has_sparse_bitmasks = ecx.split.noncont;
>  	r->alloc_capable = true;

Using resctrl_get_default_ctrl() in the architecture code like this seems awkward in
the way that the caller depends on resctrl_get_default_ctrl() returning a bitmask, thus
requiring caller to be familiar with internals of function called.

> @@ -329,7 +327,7 @@ static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r)
>  		return MAX_MBA_BW - bw;
>  
>  	pr_warn_once("Non Linear delay-bw map not supported but queried\n");
> -	return r->default_ctrl;
> +	return resctrl_get_default_ctrl(r);

I wonder if returning MAX_MBA_BW directly would not be more appropriate here ...
or returning r->membw.max_bw and doing so in previous patch.

>  }
>  
>  static void mba_wrmsr_intel(struct msr_param *m)
> @@ -438,7 +436,7 @@ static void setup_default_ctrlval(struct rdt_resource *r, u32 *dc)
>  	 * For Memory Allocation: Set b/w requested to 100%
>  	 */
>  	for (i = 0; i < hw_res->num_closid; i++, dc++)
> -		*dc = r->default_ctrl;
> +		*dc = resctrl_get_default_ctrl(r);
>  }
>  
>  static void ctrl_domain_free(struct rdt_hw_ctrl_domain *hw_dom)
> diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
> index 23a01eaebd58..5d87f279085f 100644
> --- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
> +++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
> @@ -113,8 +113,9 @@ static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s,
>   */
>  static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
>  {
> -	unsigned long first_bit, zero_bit, val;
> +	u32 supported_bits = BIT_MASK(r->cache.cbm_len) - 1;

What is criteria for caller to decide between using resctrl_get_default_ctrl() or
computing the bitmask self? Most callers already seem to be using
resctrl_get_default_ctrl() with clear expectation that it will return
a bitmask or not so it is not obvious why some callers needing bitmask
use resctrl_get_default_ctrl() while this caller compute bitmask self.

>  	unsigned int cbm_len = r->cache.cbm_len;
> +	unsigned long first_bit, zero_bit, val;
>  	int ret;
>  
>  	ret = kstrtoul(buf, 16, &val);
> @@ -123,7 +124,7 @@ static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r)
>  		return false;
>  	}
>  
> -	if ((r->cache.min_cbm_bits > 0 && val == 0) || val > r->default_ctrl) {
> +	if ((r->cache.min_cbm_bits > 0 && val == 0) || val > supported_bits) {
>  		rdt_last_cmd_puts("Mask out of range\n");
>  		return false;
>  	}
> diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
> index 1e0bae1a9d95..cd8f65c12124 100644
> --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c
> +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
> @@ -978,7 +978,7 @@ static int rdt_default_ctrl_show(struct kernfs_open_file *of,
>  	struct resctrl_schema *s = of->kn->parent->priv;
>  	struct rdt_resource *r = s->res;
>  
> -	seq_printf(seq, "%x\n", r->default_ctrl);
> +	seq_printf(seq, "%x\n", resctrl_get_default_ctrl(r));
>  	return 0;
>  }

While the function is "rdt_default_ctrl_show()" the file is "cbm_mask"
and so here also resctrl_get_default_ctrl() is implicitly assumed to
return only a bitmask.

>  
> @@ -2882,7 +2882,7 @@ static int reset_all_ctrls(struct rdt_resource *r)
>  		hw_dom = resctrl_to_arch_ctrl_dom(d);
>  
>  		for (i = 0; i < hw_res->num_closid; i++)
> -			hw_dom->ctrl_val[i] = r->default_ctrl;
> +			hw_dom->ctrl_val[i] = resctrl_get_default_ctrl(r);
>  		msr_param.dom = d;
>  		smp_call_function_any(&d->hdr.cpu_mask, rdt_ctrl_update, &msr_param, 1);
>  	}
> @@ -3417,7 +3417,7 @@ static void rdtgroup_init_mba(struct rdt_resource *r, u32 closid)
>  		}
>  
>  		cfg = &d->staged_config[CDP_NONE];
> -		cfg->new_ctrl = r->default_ctrl;
> +		cfg->new_ctrl = resctrl_get_default_ctrl(r);
>  		cfg->have_new_ctrl = true;
>  	}
>  }

Using resctrl_get_default_ctrl() only seems appropriate when setting or staging
the register values where the value returned is not further manipulated with
assumptions regarding its format.

> diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h
> index cfe451ae6ded..a939c0cec7fe 100644
> --- a/include/linux/resctrl.h
> +++ b/include/linux/resctrl.h
> @@ -216,7 +216,6 @@ enum resctrl_schema_fmt {
>   * @ctrl_domains:	RCU list of all control domains for this resource
>   * @mon_domains:	RCU list of all monitor domains for this resource
>   * @name:		Name to use in "schemata" file.
> - * @default_ctrl:	Specifies default cache cbm or memory B/W percent.
>   * @schema_fmt:		Which format string and parser is used for this schema.
>   * @evt_list:		List of monitoring events
>   * @cdp_capable:	Is the CDP feature available on this resource
> @@ -233,7 +232,6 @@ struct rdt_resource {
>  	struct list_head	ctrl_domains;
>  	struct list_head	mon_domains;
>  	char			*name;
> -	u32			default_ctrl;
>  	enum resctrl_schema_fmt	schema_fmt;
>  	struct list_head	evt_list;
>  	bool			cdp_capable;
> @@ -268,6 +266,23 @@ struct resctrl_schema {
>  	u32				num_closid;
>  };
>  
> +/**
> + * resctrl_get_default_ctrl() - Return the default control value for this
> + *                              resource.
> + * @r:		The resource whose default control type is queried.
> + */
> +static inline u32 resctrl_get_default_ctrl(struct rdt_resource *r)
> +{
> +	switch (r->schema_fmt) {
> +	case RESCTRL_SCHEMA_BITMAP:
> +		return BIT_MASK(r->cache.cbm_len) - 1;
> +	case RESCTRL_SCHEMA_RANGE:
> +		return r->membw.max_bw;
> +	}
> +
> +	return WARN_ON_ONCE(1);
> +}
> +
>  /* The number of closid supported by this resource regardless of CDP */
>  u32 resctrl_arch_get_num_closid(struct rdt_resource *r);
>  u32 resctrl_arch_system_num_rmid_idx(void);

Reinette