Re: [PATCH v5 01/27] cxl: add type2 device basic support

[Alison Schofield <alison.schofield@intel.com>]

On Fri, Nov 22, 2024 at 09:27:34AM +0000, Alejandro Lucero Palau wrote:
> 
> On 11/22/24 04:35, Alison Schofield wrote:
> > On Mon, Nov 18, 2024 at 04:44:08PM +0000, alejandro.lucero-palau@amd.com wrote:
> > > From: Alejandro Lucero <alucerop@amd.com>
> > > 
> > > Differentiate Type3, aka memory expanders, from Type2, aka device
> > > accelerators, with a new function for initializing cxl_dev_state.
> > > 
> > > Create accessors to cxl_dev_state to be used by accel drivers.
> > > 
> > > Based on previous work by Dan Williams [1]
> > > 
> > > Link: [1] https://lore.kernel.org/linux-cxl/168592160379.1948938.12863272903570476312.stgit@dwillia2-xfh.jf.intel.com/
> > > Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> > > Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> > > ---
> > >   drivers/cxl/core/memdev.c | 51 +++++++++++++++++++++++++++++++++++++++
> > >   drivers/cxl/core/pci.c    |  1 +
> > >   drivers/cxl/cxlpci.h      | 16 ------------
> > >   drivers/cxl/pci.c         | 13 +++++++---
> > >   include/cxl/cxl.h         | 21 ++++++++++++++++
> > As I mentioned in the cover letter, beginning w the first patch
> > I have depmod issues building with the cxl-test module.  I didn't
> > get very far figuring it out, other than a work-around of moving
> > the contents of include/cxl/cxl.h down into drivers/cxl/cxlmem.h.
> > That band-aid got me a bit further. In fact I wasn't so concerned
> > with breaking sfx as I was with regression testing the changes to
> > drivers/cxl/.
> > 
> > Please see if you can get the cxl-test module working again.
> 
> 
> Hi Allison,
> 
> 
> I have no problems building tools/testing/cxl and I can see cxl_test.ko in
> tools/testing/cxl/test
>

Yes that's the one. It builds it just won't load because of the
circular dependency.  Sorry I haven't been able to dig into it
further. I use run_qemu.sh [1] which uses mkosi to build the image.
It fails at the depmod step and I haven't been able to dig further.

depmod: ERROR: Cycle detected: cxl_mock -> cxl_core -> cxl_mock
depmod: ERROR: Found 2 modules in dependency cycles!

So, I'd expect it would fail at make modules_intall for you.

BTW - this happens occasionally, but usually on a smaller scale,
ie we know exactly what just changed. I suspect it happens with
only Patch 1 applied - but even limiting it to that I could not
nail it down.

--Alison


[1] https://github.com/pmem/run_qemu

> 
> I did try with the full patchset applied over 6.12-rc7 tag, and also with
> only the first patch since I was not sure if you meant the build after each
> patch is tried, but both worked once I modified the config for the checks
> inside config_check.c not to fail.
> 
> 
> I guess you meant this cxl test and not the one related to  "git clone
> https://github.com/moking/cxl-test-tool.git" what I have no experience with.
no



> 
> 
> Could someone else try this as well?
> 
> 
> > 
> > >   include/cxl/pci.h         | 23 ++++++++++++++++++
> > >   6 files changed, 105 insertions(+), 20 deletions(-)
> > >   create mode 100644 include/cxl/cxl.h
> > >   create mode 100644 include/cxl/pci.h
> > > 
> > > diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> > > index 84fefb76dafa..d083fd13a6dd 100644
> > > --- a/drivers/cxl/core/memdev.c
> > > +++ b/drivers/cxl/core/memdev.c
> > > @@ -1,6 +1,7 @@
> > >   // SPDX-License-Identifier: GPL-2.0-only
> > >   /* Copyright(c) 2020 Intel Corporation. */
> > > +#include <cxl/cxl.h>
> > >   #include <linux/io-64-nonatomic-lo-hi.h>
> > >   #include <linux/firmware.h>
> > >   #include <linux/device.h>
> > > @@ -616,6 +617,25 @@ static void detach_memdev(struct work_struct *work)
> > >   static struct lock_class_key cxl_memdev_key;
> > > +struct cxl_dev_state *cxl_accel_state_create(struct device *dev)
> > > +{
> > > +	struct cxl_dev_state *cxlds;
> > > +
> > > +	cxlds = kzalloc(sizeof(*cxlds), GFP_KERNEL);
> > > +	if (!cxlds)
> > > +		return ERR_PTR(-ENOMEM);
> > > +
> > > +	cxlds->dev = dev;
> > > +	cxlds->type = CXL_DEVTYPE_DEVMEM;
> > > +
> > > +	cxlds->dpa_res = DEFINE_RES_MEM_NAMED(0, 0, "dpa");
> > > +	cxlds->ram_res = DEFINE_RES_MEM_NAMED(0, 0, "ram");
> > > +	cxlds->pmem_res = DEFINE_RES_MEM_NAMED(0, 0, "pmem");
> > > +
> > > +	return cxlds;
> > > +}
> > > +EXPORT_SYMBOL_NS_GPL(cxl_accel_state_create, CXL);
> > > +
> > >   static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
> > >   					   const struct file_operations *fops)
> > >   {
> > > @@ -693,6 +713,37 @@ static int cxl_memdev_open(struct inode *inode, struct file *file)
> > >   	return 0;
> > >   }
> > > +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec)
> > > +{
> > > +	cxlds->cxl_dvsec = dvsec;
> > > +}
> > > +EXPORT_SYMBOL_NS_GPL(cxl_set_dvsec, CXL);
> > > +
> > > +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial)
> > > +{
> > > +	cxlds->serial = serial;
> > > +}
> > > +EXPORT_SYMBOL_NS_GPL(cxl_set_serial, CXL);
> > > +
> > > +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> > > +		     enum cxl_resource type)
> > > +{
> > > +	switch (type) {
> > > +	case CXL_RES_DPA:
> > > +		cxlds->dpa_res = res;
> > > +		return 0;
> > > +	case CXL_RES_RAM:
> > > +		cxlds->ram_res = res;
> > > +		return 0;
> > > +	case CXL_RES_PMEM:
> > > +		cxlds->pmem_res = res;
> > > +		return 0;
> > > +	}
> > > +
> > > +	return -EINVAL;
> > > +}
> > > +EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
> > > +
> > >   static int cxl_memdev_release_file(struct inode *inode, struct file *file)
> > >   {
> > >   	struct cxl_memdev *cxlmd =
> > > diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> > > index 420e4be85a1f..ff266e91ea71 100644
> > > --- a/drivers/cxl/core/pci.c
> > > +++ b/drivers/cxl/core/pci.c
> > > @@ -1,5 +1,6 @@
> > >   // SPDX-License-Identifier: GPL-2.0-only
> > >   /* Copyright(c) 2021 Intel Corporation. All rights reserved. */
> > > +#include <cxl/pci.h>
> > >   #include <linux/units.h>
> > >   #include <linux/io-64-nonatomic-lo-hi.h>
> > >   #include <linux/device.h>
> > > diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
> > > index 4da07727ab9c..eb59019fe5f3 100644
> > > --- a/drivers/cxl/cxlpci.h
> > > +++ b/drivers/cxl/cxlpci.h
> > > @@ -14,22 +14,6 @@
> > >    */
> > >   #define PCI_DVSEC_HEADER1_LENGTH_MASK	GENMASK(31, 20)
> > > -/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
> > > -#define CXL_DVSEC_PCIE_DEVICE					0
> > > -#define   CXL_DVSEC_CAP_OFFSET		0xA
> > > -#define     CXL_DVSEC_MEM_CAPABLE	BIT(2)
> > > -#define     CXL_DVSEC_HDM_COUNT_MASK	GENMASK(5, 4)
> > > -#define   CXL_DVSEC_CTRL_OFFSET		0xC
> > > -#define     CXL_DVSEC_MEM_ENABLE	BIT(2)
> > > -#define   CXL_DVSEC_RANGE_SIZE_HIGH(i)	(0x18 + (i * 0x10))
> > > -#define   CXL_DVSEC_RANGE_SIZE_LOW(i)	(0x1C + (i * 0x10))
> > > -#define     CXL_DVSEC_MEM_INFO_VALID	BIT(0)
> > > -#define     CXL_DVSEC_MEM_ACTIVE	BIT(1)
> > > -#define     CXL_DVSEC_MEM_SIZE_LOW_MASK	GENMASK(31, 28)
> > > -#define   CXL_DVSEC_RANGE_BASE_HIGH(i)	(0x20 + (i * 0x10))
> > > -#define   CXL_DVSEC_RANGE_BASE_LOW(i)	(0x24 + (i * 0x10))
> > > -#define     CXL_DVSEC_MEM_BASE_LOW_MASK	GENMASK(31, 28)
> > > -
> > >   #define CXL_DVSEC_RANGE_MAX		2
> > >   /* CXL 2.0 8.1.4: Non-CXL Function Map DVSEC */
> > > diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> > > index 188412d45e0d..0b910ef52db7 100644
> > > --- a/drivers/cxl/pci.c
> > > +++ b/drivers/cxl/pci.c
> > > @@ -1,5 +1,7 @@
> > >   // SPDX-License-Identifier: GPL-2.0-only
> > >   /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
> > > +#include <cxl/cxl.h>
> > > +#include <cxl/pci.h>
> > >   #include <linux/unaligned.h>
> > >   #include <linux/io-64-nonatomic-lo-hi.h>
> > >   #include <linux/moduleparam.h>
> > > @@ -816,6 +818,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> > >   	struct cxl_memdev *cxlmd;
> > >   	int i, rc, pmu_count;
> > >   	bool irq_avail;
> > > +	u16 dvsec;
> > >   	/*
> > >   	 * Double check the anonymous union trickery in struct cxl_regs
> > > @@ -836,13 +839,15 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> > >   	pci_set_drvdata(pdev, cxlds);
> > >   	cxlds->rcd = is_cxl_restricted(pdev);
> > > -	cxlds->serial = pci_get_dsn(pdev);
> > > -	cxlds->cxl_dvsec = pci_find_dvsec_capability(
> > > -		pdev, PCI_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
> > > -	if (!cxlds->cxl_dvsec)
> > > +	cxl_set_serial(cxlds, pci_get_dsn(pdev));
> > > +	dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
> > > +					  CXL_DVSEC_PCIE_DEVICE);
> > > +	if (!dvsec)
> > >   		dev_warn(&pdev->dev,
> > >   			 "Device DVSEC not present, skip CXL.mem init\n");
> > > +	cxl_set_dvsec(cxlds, dvsec);
> > > +
> > >   	rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
> > >   	if (rc)
> > >   		return rc;
> > > diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h
> > > new file mode 100644
> > > index 000000000000..19e5d883557a
> > > --- /dev/null
> > > +++ b/include/cxl/cxl.h
> > > @@ -0,0 +1,21 @@
> > > +/* SPDX-License-Identifier: GPL-2.0 */
> > > +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
> > > +
> > > +#ifndef __CXL_H
> > > +#define __CXL_H
> > > +
> > > +#include <linux/ioport.h>
> > > +
> > > +enum cxl_resource {
> > > +	CXL_RES_DPA,
> > > +	CXL_RES_RAM,
> > > +	CXL_RES_PMEM,
> > > +};
> > > +
> > > +struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
> > > +
> > > +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
> > > +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
> > > +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> > > +		     enum cxl_resource);
> > > +#endif
> > > diff --git a/include/cxl/pci.h b/include/cxl/pci.h
> > > new file mode 100644
> > > index 000000000000..ad63560caa2c
> > > --- /dev/null
> > > +++ b/include/cxl/pci.h
> > > @@ -0,0 +1,23 @@
> > > +/* SPDX-License-Identifier: GPL-2.0-only */
> > > +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */
> > > +
> > > +#ifndef __CXL_ACCEL_PCI_H
> > > +#define __CXL_ACCEL_PCI_H
> > > +
> > > +/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
> > > +#define CXL_DVSEC_PCIE_DEVICE					0
> > > +#define   CXL_DVSEC_CAP_OFFSET		0xA
> > > +#define     CXL_DVSEC_MEM_CAPABLE	BIT(2)
> > > +#define     CXL_DVSEC_HDM_COUNT_MASK	GENMASK(5, 4)
> > > +#define   CXL_DVSEC_CTRL_OFFSET		0xC
> > > +#define     CXL_DVSEC_MEM_ENABLE	BIT(2)
> > > +#define   CXL_DVSEC_RANGE_SIZE_HIGH(i)	(0x18 + ((i) * 0x10))
> > > +#define   CXL_DVSEC_RANGE_SIZE_LOW(i)	(0x1C + ((i) * 0x10))
> > > +#define     CXL_DVSEC_MEM_INFO_VALID	BIT(0)
> > > +#define     CXL_DVSEC_MEM_ACTIVE	BIT(1)
> > > +#define     CXL_DVSEC_MEM_SIZE_LOW_MASK	GENMASK(31, 28)
> > > +#define   CXL_DVSEC_RANGE_BASE_HIGH(i)	(0x20 + ((i) * 0x10))
> > > +#define   CXL_DVSEC_RANGE_BASE_LOW(i)	(0x24 + ((i) * 0x10))
> > > +#define     CXL_DVSEC_MEM_BASE_LOW_MASK	GENMASK(31, 28)
> > > +
> > > +#endif
> > > -- 
> > > 2.17.1
> > > 
> > >