Re: [PATCH v7] cxl/region: check interleave capability

[Alison Schofield <alison.schofield@intel.com>]

On Wed, Jun 12, 2024 at 10:04:11AM -0700, Alison Schofield wrote:
> On Tue, Jun 11, 2024 at 11:25:44PM -0400, Yao Xingtao wrote:
> > Since interleave capability is not verified, if the interleave
> > capability of a target does not match the region need, committing decoder
> > should have failed at the device end.
> 
> This needs some fixups to pass the cxl unit tests. 

BTW - not saying anything is broken in this code. It just doesn't
consider the cxl-test module and fails the unit tests. The cxl-test
module needs to mock this new stuff.

> 
> snip...
> 
> >  
> > +static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig)
> > +{
> > +	struct cxl_port *port = to_cxl_port(cxld->dev.parent);
> > +	struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);
> 
> Tried this out with cxl-test and NULL ptr deref trying to load
> the cxl-test module. Needs something like this:
> 
> diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c
> index 3482248aa344..f7ed3dd19992 100644
> --- a/tools/testing/cxl/test/cxl.c
> +++ b/tools/testing/cxl/test/cxl.c
> @@ -630,11 +630,13 @@ static struct cxl_hdm *mock_cxl_setup_hdm(struct cxl_port *port,
>                                           struct cxl_endpoint_dvsec_info *info)
>  {
>         struct cxl_hdm *cxlhdm = devm_kzalloc(&port->dev, sizeof(*cxlhdm), GFP_KERNEL);
> +       struct device *dev = &port->dev;
> 
>         if (!cxlhdm)
>                 return ERR_PTR(-ENOMEM);
> 
>         cxlhdm->port = port;
> +       dev_set_drvdata(dev, cxlhdm);
>         return cxlhdm;
>  }
> 
> 
> After that, we do load the cxl-test module but the autoconf region
> fails to set up and other unit tests fail trying to setup regions.
> I didn't go further into those, seems all failing here:
> 
> cxl_region_attach()
> 	dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n",
>                        dev_name(&cxled->cxld.dev), p->interleave_ways,
>                        p->interleave_granularity);
> 
> 
> -- Alison
> 
> > +	unsigned int interleave_mask;
> > +	u8 eiw;
> > +	u16 eig;
> > +	int rc, high_pos, low_pos;
> > +
> > +	rc = ways_to_eiw(iw, &eiw);
> > +	if (rc)
> > +		return rc;
> > +
> > +	if (!test_bit(iw, &cxlhdm->iw_cap_mask))
> > +		return -ENXIO;
> > +
> > +	rc = granularity_to_eig(ig, &eig);
> > +	if (rc)
> > +		return rc;
> > +
> > +	/*
> > +	 * Per CXL specification r3.1(8.2.4.20.13 Decoder Protection),
> > +	 * if eiw < 8:
> > +	 *   DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw]
> > +	 *   DPAOFFSET[eig + 7: 0]  = HPAOFFSET[eig + 7: 0]
> > +	 *
> > +	 *   when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the
> > +	 *   interleave bits are none.
> > +	 *
> > +	 * if eiw >= 8:
> > +	 *   DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3
> > +	 *   DPAOFFSET[eig + 7: 0]  = HPAOFFSET[eig + 7: 0]
> > +	 *
> > +	 *   when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the
> > +	 *   interleave bits are none.
> > +	 */
> > +	if (eiw == 0 || eiw == 8)
> > +		return 0;
> > +
> > +	if (eiw > 8)
> > +		high_pos = eiw + eig - 1;
> > +	else
> > +		high_pos = eiw + eig + 7;
> > +	low_pos = eig + 8;
> > +	interleave_mask = GENMASK(high_pos, low_pos);
> > +	if (interleave_mask & ~cxlhdm->interleave_mask)
> > +		return -ENXIO;
> > +
> > +	return 0;
> > +}
> > +
> >  static int cxl_port_setup_targets(struct cxl_port *port,
> >  				  struct cxl_region *cxlr,
> >  				  struct cxl_endpoint_decoder *cxled)
> > @@ -1360,6 +1431,15 @@ static int cxl_port_setup_targets(struct cxl_port *port,
> >  			return -ENXIO;
> >  		}
> >  	} else {
> > +		rc = check_interleave_cap(cxld, iw, ig);
> > +		if (rc) {
> > +			dev_dbg(&cxlr->dev,
> > +				"%s:%s iw: %d ig: %d is not supported\n",
> > +				dev_name(port->uport_dev),
> > +				dev_name(&port->dev), iw, ig);
> > +			return rc;
> > +		}
> > +
> >  		cxld->interleave_ways = iw;
> >  		cxld->interleave_granularity = ig;
> >  		cxld->hpa_range = (struct range) {
> > @@ -1796,6 +1876,15 @@ static int cxl_region_attach(struct cxl_region *cxlr,
> >  	struct cxl_dport *dport;
> >  	int rc = -ENXIO;
> >  
> > +	rc = check_interleave_cap(&cxled->cxld, p->interleave_ways,
> > +				  p->interleave_granularity);
> > +	if (rc) {
> > +		dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n",
> > +			dev_name(&cxled->cxld.dev), p->interleave_ways,
> > +			p->interleave_granularity);
> > +		return rc;
> > +	}
> > +
> >  	if (cxled->mode != cxlr->mode) {
> >  		dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
> >  			dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
> > diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
> > index 036d17db68e0..dc8e46a1fe82 100644
> > --- a/drivers/cxl/cxl.h
> > +++ b/drivers/cxl/cxl.h
> > @@ -45,6 +45,8 @@
> >  #define   CXL_HDM_DECODER_TARGET_COUNT_MASK GENMASK(7, 4)
> >  #define   CXL_HDM_DECODER_INTERLEAVE_11_8 BIT(8)
> >  #define   CXL_HDM_DECODER_INTERLEAVE_14_12 BIT(9)
> > +#define   CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY BIT(11)
> > +#define   CXL_HDM_DECODER_INTERLEAVE_16_WAY BIT(12)
> >  #define CXL_HDM_DECODER_CTRL_OFFSET 0x4
> >  #define   CXL_HDM_DECODER_ENABLE BIT(1)
> >  #define CXL_HDM_DECODER0_BASE_LOW_OFFSET(i) (0x20 * (i) + 0x10)
> > diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> > index 36cee9c30ceb..7fe617122d33 100644
> > --- a/drivers/cxl/cxlmem.h
> > +++ b/drivers/cxl/cxlmem.h
> > @@ -848,11 +848,21 @@ static inline void cxl_mem_active_dec(void)
> >  
> >  int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd);
> >  
> > +/*
> > + * struct cxl_hdm - HDM Decoder registers and cached / decoded capabilities
> > + * @regs: mapped registers, see devm_cxl_setup_hdm()
> > + * @decoder_count: number of decoders for this port
> > + * @target_count: for switch decoders, max downstream port targets
> > + * @interleave_mask: interleave granularity capability, see check_interleave_cap()
> > + * @iw_cap_mask: bitmask of supported interleave ways, see check_interleave_cap()
> > + * @port: mapped cxl_port, see devm_cxl_setup_hdm()
> > + */
> >  struct cxl_hdm {
> >  	struct cxl_component_regs regs;
> >  	unsigned int decoder_count;
> >  	unsigned int target_count;
> >  	unsigned int interleave_mask;
> > +	unsigned long iw_cap_mask;
> >  	struct cxl_port *port;
> >  };
> >  
> > -- 
> > 2.37.3
> > 
>