Re: [PATCH v3 06/25] cxl/mem: Read dynamic capacity configuration from the device

[Ira Weiny <ira.weiny@intel.com>]

Fan Ni wrote:
> On Fri, Aug 16, 2024 at 02:45:47PM -0700, Dave Jiang wrote:
> > 
> > > +
> > > +/**
> > > + * cxl_dev_dynamic_capacity_identify() - Reads the dynamic capacity
> > > + *					 information from the device.
> > > + * @mds: The memory device state
> > > + *
> > > + * Read Dynamic Capacity information from the device and populate the state
> > > + * structures for later use.
> > > + *
> > > + * Return: 0 if identify was executed successfully, -ERRNO on error.
> > > + */
> > > +int cxl_dev_dynamic_capacity_identify(struct cxl_memdev_state *mds)
> > > +{
> > > +	size_t dc_resp_size = mds->payload_size;
> > > +	struct device *dev = mds->cxlds.dev;
> > > +	u8 start_region, i;
> > > +
> > > +	for (i = 0; i < CXL_MAX_DC_REGION; i++)
> > > +		snprintf(mds->dc_region[i].name, CXL_DC_REGION_STRLEN, "<nil>");
> > > +
> > > +	if (!cxl_dcd_supported(mds)) {
> > > +		dev_dbg(dev, "DCD not supported\n");
> > > +		return 0;
> > > +	}
> > 
> > This should happen before you pre-format the name string? I would assume that if DCD is not supported then the dcd name sysfs attribs would be not be visible?
> > 

No this string is not used for sysfs.  It is used to label the dpa
resources...  That said in review I don't recall why it was necessary to
add the '<nil>' to them by default.  I'm actually going to remove that and
continue testing and if I recall where this was showing up I might add it
back in.

> > > +
> > > +	struct cxl_mbox_get_dc_config_out *dc_resp __free(kfree) =
> > > +					kvmalloc(dc_resp_size, GFP_KERNEL);
> > > +	if (!dc_resp)
> > > +		return -ENOMEM;
> > > +
> > > +	start_region = 0;
> > > +	do {
> > > +		int rc, j;
> > > +
> > > +		rc = cxl_get_dc_config(mds, start_region, dc_resp, dc_resp_size);
> > > +		if (rc < 0) {
> > > +			dev_dbg(dev, "Failed to get DC config: %d\n", rc);
> > > +			return rc;
> > > +		}
> > > +
> > > +		mds->nr_dc_region += rc;
> > > +
> > > +		if (mds->nr_dc_region < 1 || mds->nr_dc_region > CXL_MAX_DC_REGION) {
> > > +			dev_err(dev, "Invalid num of dynamic capacity regions %d\n",
> > > +				mds->nr_dc_region);
> > > +			return -EINVAL;
> > > +		}
> > > +
> > > +		for (i = start_region, j = 0; i < mds->nr_dc_region; i++, j++) {
> > 
> > This should be 'j < mds->nr_dc_region'? Otherwise if your start region say is '3' and you have '2' DC regions, you never enter the loop. Or does that not happen? I also wonder if you need to check if 'start_region + mds->nr_dc_region > CXL_MAX_DC_REGION'.
> > 
> That can not happen, start_region was updated to the number of regions
> has returned till now (not counting the current call), while
> nr_dc_region is the total number of regions returned till now (including
> the current call) as we update it above, so start_region should never be larger
> than nr_dc_region.

Yep.

> 
> > > +			rc = cxl_dc_save_region_info(mds, i, &dc_resp->region[j]);
> > > +			if (rc) {
> > > +				dev_dbg(dev, "Failed to save region info: %d\n", rc);
> 
> I am not sure why we sometimes use dev_err and sometimes we use dev_dbg
> here, if dcd is supported, error from getting dc configuration is an
> error to me.

We are trying to reduce the dev_err() use.  cxl_dc_save_region_info() has
dev_err() which is much more specific as to the error.  At worse this is
just redundant as a debug.

I'll remove it because the debug output is pretty verbose too.

Ira

> 
> Fan
> 
> > > +				return rc;
> > > +			}
> > > +		}
> > > +
> > > +		start_region = mds->nr_dc_region;
> > > +
> > > +	} while (mds->nr_dc_region < dc_resp->avail_region_count);
> > > +
> > > +	mds->dynamic_bytes =
> > > +		mds->dc_region[mds->nr_dc_region - 1].base +
> > > +		mds->dc_region[mds->nr_dc_region - 1].decode_len -
> > > +		mds->dc_region[0].base;
> > > +	dev_dbg(dev, "Total dynamic range: %#llx\n", mds->dynamic_bytes);
> > > +
> > > +	return 0;
> > > +}
> > > +EXPORT_SYMBOL_NS_GPL(cxl_dev_dynamic_capacity_identify, CXL);
> > > +
> > >  static int add_dpa_res(struct device *dev, struct resource *parent,
> > >  		       struct resource *res, resource_size_t start,
> > >  		       resource_size_t size, const char *type)
> > > @@ -1294,8 +1447,15 @@ int cxl_mem_create_range_info(struct cxl_memdev_state *mds)
> > >  {
> > >  	struct cxl_dev_state *cxlds = &mds->cxlds;
> > >  	struct device *dev = cxlds->dev;
> > > +	size_t untenanted_mem;
> > >  	int rc;
> > >  
> > > +	mds->total_bytes = mds->static_bytes;
> > > +	if (mds->nr_dc_region) {
> > > +		untenanted_mem = mds->dc_region[0].base - mds->static_bytes;
> > > +		mds->total_bytes += untenanted_mem + mds->dynamic_bytes;
> > > +	}
> > > +
> > >  	if (!cxlds->media_ready) {
> > >  		cxlds->dpa_res = DEFINE_RES_MEM(0, 0);
> > >  		cxlds->ram_res = DEFINE_RES_MEM(0, 0);
> > > @@ -1305,6 +1465,15 @@ int cxl_mem_create_range_info(struct cxl_memdev_state *mds)
> > >  
> > >  	cxlds->dpa_res = DEFINE_RES_MEM(0, mds->total_bytes);
> > >  
> > > +	for (int i = 0; i < mds->nr_dc_region; i++) {
> > > +		struct cxl_dc_region_info *dcr = &mds->dc_region[i];
> > > +
> > > +		rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->dc_res[i],
> > > +				 dcr->base, dcr->decode_len, dcr->name);
> > > +		if (rc)
> > > +			return rc;
> > > +	}
> > > +
> > >  	if (mds->partition_align_bytes == 0) {
> > >  		rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
> > >  				 mds->volatile_only_bytes, "ram");
> > > diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> > > index f2f8b567e0e7..b4eb8164d05d 100644
> > > --- a/drivers/cxl/cxlmem.h
> > > +++ b/drivers/cxl/cxlmem.h
> > > @@ -402,6 +402,7 @@ enum cxl_devtype {
> > >  	CXL_DEVTYPE_CLASSMEM,
> > >  };
> > >  
> > > +#define CXL_MAX_DC_REGION 8
> > >  /**
> > >   * struct cxl_dpa_perf - DPA performance property entry
> > >   * @dpa_range: range for DPA address
> > > @@ -431,6 +432,8 @@ struct cxl_dpa_perf {
> > >   * @dpa_res: Overall DPA resource tree for the device
> > >   * @pmem_res: Active Persistent memory capacity configuration
> > >   * @ram_res: Active Volatile memory capacity configuration
> > > + * @dc_res: Active Dynamic Capacity memory configuration for each possible
> > > + *          region
> > >   * @serial: PCIe Device Serial Number
> > >   * @type: Generic Memory Class device or Vendor Specific Memory device
> > >   */
> > > @@ -445,10 +448,22 @@ struct cxl_dev_state {
> > >  	struct resource dpa_res;
> > >  	struct resource pmem_res;
> > >  	struct resource ram_res;
> > > +	struct resource dc_res[CXL_MAX_DC_REGION];
> > >  	u64 serial;
> > >  	enum cxl_devtype type;
> > >  };
> > >  
> > > +#define CXL_DC_REGION_STRLEN > +struct cxl_dc_region_info {
> > > +	u64 base;
> > > +	u64 decode_len;
> > > +	u64 len;
> > > +	u64 blk_size;
> > > +	u32 dsmad_handle;
> > > +	u8 flags;
> > > +	u8 name[CXL_DC_REGION_STRLEN];
> > > +};
> > 
> > Does this need kdoc comments?
> > 
> > 
> > > +
> > >  /**
> > >   * struct cxl_memdev_state - Generic Type-3 Memory Device Class driver data
> > >   *
> > > @@ -466,7 +481,9 @@ struct cxl_dev_state {
> > >   * @dcd_cmds: List of DCD commands implemented by memory device
> > >   * @enabled_cmds: Hardware commands found enabled in CEL.
> > >   * @exclusive_cmds: Commands that are kernel-internal only
> > > - * @total_bytes: sum of all possible capacities
> > > + * @total_bytes: length of all possible capacities
> > > + * @static_bytes: length of possible static RAM and PMEM partitions
> > > + * @dynamic_bytes: length of possible DC partitions (DC Regions)
> > 
> > Did this get added to the wrong struct comment header? 'cxl_dev_state' instead of 'cxl_memdev_state'?
> > >   * @volatile_only_bytes: hard volatile capacity
> > >   * @persistent_only_bytes: hard persistent capacity
> > >   * @partition_align_bytes: alignment size for partition-able capacity
> > > @@ -476,6 +493,8 @@ struct cxl_dev_state {
> > >   * @next_persistent_bytes: persistent capacity change pending device reset
> > >   * @ram_perf: performance data entry matched to RAM partition
> > >   * @pmem_perf: performance data entry matched to PMEM partition
> > > + * @nr_dc_region: number of DC regions implemented in the memory device
> > > + * @dc_region: array containing info about the DC regions
> > Did this get added to the wrong struct comment header? 'cxl_dev_state' instead of 'cxl_memdev_state'?
> > 
> > DJ
> > 
> > >   * @event: event log driver state
> > >   * @poison: poison driver state info
> > >   * @security: security driver state info
> > > @@ -496,6 +515,8 @@ struct cxl_memdev_state {
> > >  	DECLARE_BITMAP(enabled_cmds, CXL_MEM_COMMAND_ID_MAX);
> > >  	DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
> > >  	u64 total_bytes;
> > > +	u64 static_bytes;
> > > +	u64 dynamic_bytes;
> > >  	u64 volatile_only_bytes;
> > >  	u64 persistent_only_bytes;
> > >  	u64 partition_align_bytes;
> > > @@ -507,6 +528,9 @@ struct cxl_memdev_state {
> > >  	struct cxl_dpa_perf ram_perf;
> > >  	struct cxl_dpa_perf pmem_perf;
> > >  
> > > +	u8 nr_dc_region;
> > > +	struct cxl_dc_region_info dc_region[CXL_MAX_DC_REGION];
> > > +
> > >  	struct cxl_event_state event;
> > >  	struct cxl_poison_state poison;
> > >  	struct cxl_security_state security;
> > > @@ -709,6 +733,32 @@ struct cxl_mbox_set_partition_info {
> > >  
> > >  #define  CXL_SET_PARTITION_IMMEDIATE_FLAG	BIT(0)
> > >  
> > > +/* See CXL 3.1 Table 8-163 get dynamic capacity config Input Payload */
> > > +struct cxl_mbox_get_dc_config_in {
> > > +	u8 region_count;
> > > +	u8 start_region_index;
> > > +} __packed;
> > > +
> > > +/* See CXL 3.1 Table 8-164 get dynamic capacity config Output Payload */
> > > +struct cxl_mbox_get_dc_config_out {
> > > +	u8 avail_region_count;
> > > +	u8 regions_returned;
> > > +	u8 rsvd[6];
> > > +	/* See CXL 3.1 Table 8-165 */
> > > +	struct cxl_dc_region_config {
> > > +		__le64 region_base;
> > > +		__le64 region_decode_length;
> > > +		__le64 region_length;
> > > +		__le64 region_block_size;
> > > +		__le32 region_dsmad_handle;
> > > +		u8 flags;
> > > +		u8 rsvd[3];
> > > +	} __packed region[];
> > > +	/* Trailing fields unused */
> > > +} __packed;
> > > +#define CXL_DYNAMIC_CAPACITY_SANITIZE_ON_RELEASE_FLAG BIT(0)
> > > +#define CXL_DCD_BLOCK_LINE_SIZE 0x40
> > > +
> > >  /* Set Timestamp CXL 3.0 Spec 8.2.9.4.2 */
> > >  struct cxl_mbox_set_timestamp_in {
> > >  	__le64 timestamp;
> > > @@ -832,6 +882,7 @@ enum {
> > >  int cxl_internal_send_cmd(struct cxl_memdev_state *mds,
> > >  			  struct cxl_mbox_cmd *cmd);
> > >  int cxl_dev_state_identify(struct cxl_memdev_state *mds);
> > > +int cxl_dev_dynamic_capacity_identify(struct cxl_memdev_state *mds);
> > >  int cxl_await_media_ready(struct cxl_dev_state *cxlds);
> > >  int cxl_enumerate_cmds(struct cxl_memdev_state *mds);
> > >  int cxl_mem_create_range_info(struct cxl_memdev_state *mds);
> > > @@ -845,6 +896,17 @@ void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
> > >  			    enum cxl_event_log_type type,
> > >  			    enum cxl_event_type event_type,
> > >  			    const uuid_t *uuid, union cxl_event *evt);
> > > +
> > > +static inline bool cxl_dcd_supported(struct cxl_memdev_state *mds)
> > > +{
> > > +	return test_bit(CXL_DCD_ENABLED_GET_CONFIG, mds->dcd_cmds);
> > > +}
> > > +
> > > +static inline void cxl_disable_dcd(struct cxl_memdev_state *mds)
> > > +{
> > > +	clear_bit(CXL_DCD_ENABLED_GET_CONFIG, mds->dcd_cmds);
> > > +}
> > > +
> > >  int cxl_set_timestamp(struct cxl_memdev_state *mds);
> > >  int cxl_poison_state_init(struct cxl_memdev_state *mds);
> > >  int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
> > > diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> > > index 3a60cd66263e..f7f03599bc83 100644
> > > --- a/drivers/cxl/pci.c
> > > +++ b/drivers/cxl/pci.c
> > > @@ -874,6 +874,10 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> > >  	if (rc)
> > >  		return rc;
> > >  
> > > +	rc = cxl_dev_dynamic_capacity_identify(mds);
> > > +	if (rc)
> > > +		cxl_disable_dcd(mds);
> > > +
> > >  	rc = cxl_mem_create_range_info(mds);
> > >  	if (rc)
> > >  		return rc;
> > > 
>