* [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-07 20:26 ` kernel test robot
` (4 more replies)
2024-09-07 8:18 ` [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port alejandro.lucero-palau
` (18 subsequent siblings)
19 siblings, 5 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Differientiate Type3, aka memory expanders, from Type2, aka device
accelerators, with a new function for initializing cxl_dev_state.
Create accessors to cxl_dev_state to be used by accel drivers.
Add SFC ethernet network driver as the client.
Based on https://lore.kernel.org/linux-cxl/168592160379.1948938.12863272903570476312.stgit@dwillia2-xfh.jf.intel.com/
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
Co-developed-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/cxl/core/memdev.c | 52 ++++++++++++++++
drivers/cxl/core/pci.c | 1 +
drivers/cxl/cxlpci.h | 16 -----
drivers/cxl/pci.c | 13 ++--
drivers/net/ethernet/sfc/Makefile | 2 +-
drivers/net/ethernet/sfc/efx.c | 13 ++++
drivers/net/ethernet/sfc/efx_cxl.c | 86 +++++++++++++++++++++++++++
drivers/net/ethernet/sfc/efx_cxl.h | 29 +++++++++
drivers/net/ethernet/sfc/net_driver.h | 6 ++
include/linux/cxl/cxl.h | 21 +++++++
include/linux/cxl/pci.h | 23 +++++++
11 files changed, 241 insertions(+), 21 deletions(-)
create mode 100644 drivers/net/ethernet/sfc/efx_cxl.c
create mode 100644 drivers/net/ethernet/sfc/efx_cxl.h
create mode 100644 include/linux/cxl/cxl.h
create mode 100644 include/linux/cxl/pci.h
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 0277726afd04..10c0a6990f9a 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2020 Intel Corporation. */
+#include <linux/cxl/cxl.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/firmware.h>
#include <linux/device.h>
@@ -615,6 +616,25 @@ static void detach_memdev(struct work_struct *work)
static struct lock_class_key cxl_memdev_key;
+struct cxl_dev_state *cxl_accel_state_create(struct device *dev)
+{
+ struct cxl_dev_state *cxlds;
+
+ cxlds = kzalloc(sizeof(*cxlds), GFP_KERNEL);
+ if (!cxlds)
+ return ERR_PTR(-ENOMEM);
+
+ cxlds->dev = dev;
+ cxlds->type = CXL_DEVTYPE_DEVMEM;
+
+ cxlds->dpa_res = DEFINE_RES_MEM_NAMED(0, 0, "dpa");
+ cxlds->ram_res = DEFINE_RES_MEM_NAMED(0, 0, "ram");
+ cxlds->pmem_res = DEFINE_RES_MEM_NAMED(0, 0, "pmem");
+
+ return cxlds;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_accel_state_create, CXL);
+
static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
const struct file_operations *fops)
{
@@ -692,6 +712,38 @@ static int cxl_memdev_open(struct inode *inode, struct file *file)
return 0;
}
+void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec)
+{
+ cxlds->cxl_dvsec = dvsec;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_set_dvsec, CXL);
+
+void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial)
+{
+ cxlds->serial = serial;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_set_serial, CXL);
+
+int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
+ enum cxl_resource type)
+{
+ switch (type) {
+ case CXL_ACCEL_RES_DPA:
+ cxlds->dpa_res = res;
+ return 0;
+ case CXL_ACCEL_RES_RAM:
+ cxlds->ram_res = res;
+ return 0;
+ case CXL_ACCEL_RES_PMEM:
+ cxlds->pmem_res = res;
+ return 0;
+ default:
+ dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
+ return -EINVAL;
+ }
+}
+EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
+
static int cxl_memdev_release_file(struct inode *inode, struct file *file)
{
struct cxl_memdev *cxlmd =
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index 51132a575b27..3d6564dbda57 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -7,6 +7,7 @@
#include <linux/pci.h>
#include <linux/pci-doe.h>
#include <linux/aer.h>
+#include <linux/cxl/pci.h>
#include <cxlpci.h>
#include <cxlmem.h>
#include <cxl.h>
diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
index 4da07727ab9c..eb59019fe5f3 100644
--- a/drivers/cxl/cxlpci.h
+++ b/drivers/cxl/cxlpci.h
@@ -14,22 +14,6 @@
*/
#define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20)
-/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
-#define CXL_DVSEC_PCIE_DEVICE 0
-#define CXL_DVSEC_CAP_OFFSET 0xA
-#define CXL_DVSEC_MEM_CAPABLE BIT(2)
-#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
-#define CXL_DVSEC_CTRL_OFFSET 0xC
-#define CXL_DVSEC_MEM_ENABLE BIT(2)
-#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
-#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
-#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
-#define CXL_DVSEC_MEM_ACTIVE BIT(1)
-#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
-#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
-#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
-#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
-
#define CXL_DVSEC_RANGE_MAX 2
/* CXL 2.0 8.1.4: Non-CXL Function Map DVSEC */
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 4be35dc22202..742a7b2a1be5 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -11,6 +11,8 @@
#include <linux/pci.h>
#include <linux/aer.h>
#include <linux/io.h>
+#include <linux/cxl/cxl.h>
+#include <linux/cxl/pci.h>
#include "cxlmem.h"
#include "cxlpci.h"
#include "cxl.h"
@@ -795,6 +797,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
struct cxl_memdev *cxlmd;
int i, rc, pmu_count;
bool irq_avail;
+ u16 dvsec;
/*
* Double check the anonymous union trickery in struct cxl_regs
@@ -815,12 +818,14 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
pci_set_drvdata(pdev, cxlds);
cxlds->rcd = is_cxl_restricted(pdev);
- cxlds->serial = pci_get_dsn(pdev);
- cxlds->cxl_dvsec = pci_find_dvsec_capability(
- pdev, PCI_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
- if (!cxlds->cxl_dvsec)
+ cxl_set_serial(cxlds, pci_get_dsn(pdev));
+ dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
+ CXL_DVSEC_PCIE_DEVICE);
+ if (!dvsec)
dev_warn(&pdev->dev,
"Device DVSEC not present, skip CXL.mem init\n");
+ else
+ cxl_set_dvsec(cxlds, dvsec);
rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
if (rc)
diff --git a/drivers/net/ethernet/sfc/Makefile b/drivers/net/ethernet/sfc/Makefile
index 8f446b9bd5ee..e80c713c3b0c 100644
--- a/drivers/net/ethernet/sfc/Makefile
+++ b/drivers/net/ethernet/sfc/Makefile
@@ -7,7 +7,7 @@ sfc-y += efx.o efx_common.o efx_channels.o nic.o \
mcdi_functions.o mcdi_filters.o mcdi_mon.o \
ef100.o ef100_nic.o ef100_netdev.o \
ef100_ethtool.o ef100_rx.o ef100_tx.o \
- efx_devlink.o
+ efx_devlink.o efx_cxl.o
sfc-$(CONFIG_SFC_MTD) += mtd.o
sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o \
mae.o tc.o tc_bindings.o tc_counters.o \
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 6f1a01ded7d4..3a7406aa950c 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -33,6 +33,7 @@
#include "selftest.h"
#include "sriov.h"
#include "efx_devlink.h"
+#include "efx_cxl.h"
#include "mcdi_port_common.h"
#include "mcdi_pcol.h"
@@ -899,6 +900,9 @@ static void efx_pci_remove(struct pci_dev *pci_dev)
efx_pci_remove_main(efx);
efx_fini_io(efx);
+
+ efx_cxl_exit(efx);
+
pci_dbg(efx->pci_dev, "shutdown successful\n");
efx_fini_devlink_and_unlock(efx);
@@ -1109,6 +1113,15 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
if (rc)
goto fail2;
+ /* A successful cxl initialization implies a CXL region created to be
+ * used for PIO buffers. If there is no CXL support, or initialization
+ * fails, efx_cxl_pio_initialised wll be false and legacy PIO buffers
+ * defined at specific PCI BAR regions will be used.
+ */
+ rc = efx_cxl_init(efx);
+ if (rc)
+ pci_err(pci_dev, "CXL initialization failed with error %d\n", rc);
+
rc = efx_pci_probe_post_io(efx);
if (rc) {
/* On failure, retry once immediately.
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
new file mode 100644
index 000000000000..bba36cbbab22
--- /dev/null
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/****************************************************************************
+ *
+ * Driver for AMD network controllers and boards
+ * Copyright (C) 2024, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#include <linux/cxl/cxl.h>
+#include <linux/cxl/pci.h>
+#include <linux/pci.h>
+
+#include "net_driver.h"
+#include "efx_cxl.h"
+
+#define EFX_CTPIO_BUFFER_SIZE (1024 * 1024 * 256)
+
+int efx_cxl_init(struct efx_nic *efx)
+{
+ struct pci_dev *pci_dev = efx->pci_dev;
+ struct efx_cxl *cxl;
+ struct resource res;
+ u16 dvsec;
+ int rc;
+
+ efx->efx_cxl_pio_initialised = false;
+
+ dvsec = pci_find_dvsec_capability(pci_dev, PCI_VENDOR_ID_CXL,
+ CXL_DVSEC_PCIE_DEVICE);
+
+ if (!dvsec)
+ return 0;
+
+ pci_dbg(pci_dev, "CXL_DVSEC_PCIE_DEVICE capability found\n");
+
+ efx->cxl = kzalloc(sizeof(*cxl), GFP_KERNEL);
+ if (!efx->cxl)
+ return -ENOMEM;
+
+ cxl = efx->cxl;
+
+ cxl->cxlds = cxl_accel_state_create(&pci_dev->dev);
+ if (IS_ERR(cxl->cxlds)) {
+ pci_err(pci_dev, "CXL accel device state failed");
+ kfree(efx->cxl);
+ return -ENOMEM;
+ }
+
+ cxl_set_dvsec(cxl->cxlds, dvsec);
+ cxl_set_serial(cxl->cxlds, pci_dev->dev.id);
+
+ res = DEFINE_RES_MEM(0, EFX_CTPIO_BUFFER_SIZE);
+ if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_DPA)) {
+ pci_err(pci_dev, "cxl_set_resource DPA failed\n");
+ rc = -EINVAL;
+ goto err;
+ }
+
+ res = DEFINE_RES_MEM_NAMED(0, EFX_CTPIO_BUFFER_SIZE, "ram");
+ if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_RAM)) {
+ pci_err(pci_dev, "cxl_set_resource RAM failed\n");
+ rc = -EINVAL;
+ goto err;
+ }
+
+ return 0;
+err:
+ kfree(cxl->cxlds);
+ kfree(cxl);
+ efx->cxl = NULL;
+
+ return rc;
+}
+
+void efx_cxl_exit(struct efx_nic *efx)
+{
+ if (efx->cxl) {
+ kfree(efx->cxl->cxlds);
+ kfree(efx->cxl);
+ }
+}
+
+MODULE_IMPORT_NS(CXL);
diff --git a/drivers/net/ethernet/sfc/efx_cxl.h b/drivers/net/ethernet/sfc/efx_cxl.h
new file mode 100644
index 000000000000..f57fb2afd124
--- /dev/null
+++ b/drivers/net/ethernet/sfc/efx_cxl.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/****************************************************************************
+ * Driver for AMD network controllers and boards
+ * Copyright (C) 2024, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#ifndef EFX_CXL_H
+#define EFX_CXL_H
+
+struct efx_nic;
+struct cxl_dev_state;
+
+struct efx_cxl {
+ struct cxl_dev_state *cxlds;
+ struct cxl_memdev *cxlmd;
+ struct cxl_root_decoder *cxlrd;
+ struct cxl_port *endpoint;
+ struct cxl_endpoint_decoder *cxled;
+ struct cxl_region *efx_region;
+ void __iomem *ctpio_cxl;
+};
+
+int efx_cxl_init(struct efx_nic *efx);
+void efx_cxl_exit(struct efx_nic *efx);
+#endif
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index b85c51cbe7f9..77261de65e63 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -817,6 +817,8 @@ enum efx_xdp_tx_queues_mode {
struct efx_mae;
+struct efx_cxl;
+
/**
* struct efx_nic - an Efx NIC
* @name: Device name (net device name or bus id before net device registered)
@@ -963,6 +965,8 @@ struct efx_mae;
* @tc: state for TC offload (EF100).
* @devlink: reference to devlink structure owned by this device
* @dl_port: devlink port associated with the PF
+ * @cxl: details of related cxl objects
+ * @efx_cxl_pio_initialised: clx initialization outcome.
* @mem_bar: The BAR that is mapped into membase.
* @reg_base: Offset from the start of the bar to the function control window.
* @monitor_work: Hardware monitor workitem
@@ -1148,6 +1152,8 @@ struct efx_nic {
struct devlink *devlink;
struct devlink_port *dl_port;
+ struct efx_cxl *cxl;
+ bool efx_cxl_pio_initialised;
unsigned int mem_bar;
u32 reg_base;
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
new file mode 100644
index 000000000000..e78eefa82123
--- /dev/null
+++ b/include/linux/cxl/cxl.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
+
+#ifndef __CXL_H
+#define __CXL_H
+
+#include <linux/device.h>
+
+enum cxl_resource {
+ CXL_ACCEL_RES_DPA,
+ CXL_ACCEL_RES_RAM,
+ CXL_ACCEL_RES_PMEM,
+};
+
+struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
+
+void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
+void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
+int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
+ enum cxl_resource);
+#endif
diff --git a/include/linux/cxl/pci.h b/include/linux/cxl/pci.h
new file mode 100644
index 000000000000..c337ae8797e6
--- /dev/null
+++ b/include/linux/cxl/pci.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
+
+#ifndef __CXL_ACCEL_PCI_H
+#define __CXL_ACCEL_PCI_H
+
+/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
+#define CXL_DVSEC_PCIE_DEVICE 0
+#define CXL_DVSEC_CAP_OFFSET 0xA
+#define CXL_DVSEC_MEM_CAPABLE BIT(2)
+#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
+#define CXL_DVSEC_CTRL_OFFSET 0xC
+#define CXL_DVSEC_MEM_ENABLE BIT(2)
+#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
+#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
+#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
+#define CXL_DVSEC_MEM_ACTIVE BIT(1)
+#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
+#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
+#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
+#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
+
+#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-07 8:18 ` [PATCH v3 01/20] cxl: add type2 device basic support alejandro.lucero-palau
@ 2024-09-07 20:26 ` kernel test robot
2024-09-10 6:12 ` Li, Ming4
` (3 subsequent siblings)
4 siblings, 0 replies; 88+ messages in thread
From: kernel test robot @ 2024-09-07 20:26 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: oe-kbuild-all, Alejandro Lucero
Hi,
kernel test robot noticed the following build errors:
[auto build test ERROR on cxl/next]
[also build test ERROR on linus/master v6.11-rc6 next-20240906]
[cannot apply to cxl/pending horms-ipvs/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/alejandro-lucero-palau-amd-com/cxl-add-type2-device-basic-support/20240907-162231
base: https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git next
patch link: https://lore.kernel.org/r/20240907081836.5801-2-alejandro.lucero-palau%40amd.com
patch subject: [PATCH v3 01/20] cxl: add type2 device basic support
config: mips-ip27_defconfig (https://download.01.org/0day-ci/archive/20240908/202409080455.S5Uvi35I-lkp@intel.com/config)
compiler: mips64-linux-gcc (GCC) 14.1.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240908/202409080455.S5Uvi35I-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202409080455.S5Uvi35I-lkp@intel.com/
All errors (new ones prefixed by >>, old ones prefixed by <<):
WARNING: modpost: missing MODULE_DESCRIPTION() in fs/btrfs/btrfs.o
WARNING: modpost: missing MODULE_DESCRIPTION() in crypto/cast_common.o
WARNING: modpost: missing MODULE_DESCRIPTION() in lib/crypto/libarc4.o
WARNING: modpost: missing MODULE_DESCRIPTION() in lib/zlib_inflate/zlib_inflate.o
WARNING: modpost: missing MODULE_DESCRIPTION() in lib/zlib_deflate/zlib_deflate.o
WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/base/regmap/regmap-i2c.o
WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/mfd/mfd-core.o
WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/mfd/pcf50633-gpio.o
WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/uio/uio_aec.o
WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/cdrom/cdrom.o
WARNING: modpost: missing MODULE_DESCRIPTION() in drivers/i2c/busses/i2c-ali1563.o
>> ERROR: modpost: "cxl_accel_state_create" [drivers/net/ethernet/sfc/sfc.ko] undefined!
>> ERROR: modpost: "cxl_set_serial" [drivers/net/ethernet/sfc/sfc.ko] undefined!
ERROR: modpost: "cxl_set_resource" [drivers/net/ethernet/sfc/sfc.ko] undefined!
ERROR: modpost: "cxl_set_dvsec" [drivers/net/ethernet/sfc/sfc.ko] undefined!
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-07 8:18 ` [PATCH v3 01/20] cxl: add type2 device basic support alejandro.lucero-palau
2024-09-07 20:26 ` kernel test robot
@ 2024-09-10 6:12 ` Li, Ming4
2024-09-10 7:25 ` Alejandro Lucero Palau
2024-09-12 8:57 ` Zhi Wang
` (2 subsequent siblings)
4 siblings, 1 reply; 88+ messages in thread
From: Li, Ming4 @ 2024-09-10 6:12 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Differientiate Type3, aka memory expanders, from Type2, aka device
> accelerators, with a new function for initializing cxl_dev_state.
>
> Create accessors to cxl_dev_state to be used by accel drivers.
>
> Add SFC ethernet network driver as the client.
>
> Based on https://lore.kernel.org/linux-cxl/168592160379.1948938.12863272903570476312.stgit@dwillia2-xfh.jf.intel.com/
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> ---
> drivers/cxl/core/memdev.c | 52 ++++++++++++++++
> drivers/cxl/core/pci.c | 1 +
> drivers/cxl/cxlpci.h | 16 -----
> drivers/cxl/pci.c | 13 ++--
> drivers/net/ethernet/sfc/Makefile | 2 +-
> drivers/net/ethernet/sfc/efx.c | 13 ++++
> drivers/net/ethernet/sfc/efx_cxl.c | 86 +++++++++++++++++++++++++++
> drivers/net/ethernet/sfc/efx_cxl.h | 29 +++++++++
> drivers/net/ethernet/sfc/net_driver.h | 6 ++
> include/linux/cxl/cxl.h | 21 +++++++
> include/linux/cxl/pci.h | 23 +++++++
> 11 files changed, 241 insertions(+), 21 deletions(-)
> create mode 100644 drivers/net/ethernet/sfc/efx_cxl.c
> create mode 100644 drivers/net/ethernet/sfc/efx_cxl.h
> create mode 100644 include/linux/cxl/cxl.h
> create mode 100644 include/linux/cxl/pci.h
>
[...]
>
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> new file mode 100644
> index 000000000000..e78eefa82123
> --- /dev/null
> +++ b/include/linux/cxl/cxl.h
> @@ -0,0 +1,21 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
> +
> +#ifndef __CXL_H
> +#define __CXL_H
> +
> +#include <linux/device.h>
> +
> +enum cxl_resource {
> + CXL_ACCEL_RES_DPA,
> + CXL_ACCEL_RES_RAM,
> + CXL_ACCEL_RES_PMEM,
> +};
Can remove 'ACCEL' from the resource name? they can be used for both type-2 and type-3 devices.
> +
> +struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
> +
> +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
> +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> + enum cxl_resource);
> +#endif
> diff --git a/include/linux/cxl/pci.h b/include/linux/cxl/pci.h
> new file mode 100644
> index 000000000000..c337ae8797e6
> --- /dev/null
> +++ b/include/linux/cxl/pci.h
> @@ -0,0 +1,23 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
> +
> +#ifndef __CXL_ACCEL_PCI_H
> +#define __CXL_ACCEL_PCI_H
> +
> +/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
> +#define CXL_DVSEC_PCIE_DEVICE 0
> +#define CXL_DVSEC_CAP_OFFSET 0xA
> +#define CXL_DVSEC_MEM_CAPABLE BIT(2)
> +#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
> +#define CXL_DVSEC_CTRL_OFFSET 0xC
> +#define CXL_DVSEC_MEM_ENABLE BIT(2)
> +#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
> +#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
> +#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
> +#define CXL_DVSEC_MEM_ACTIVE BIT(1)
> +#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
> +#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
> +#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
> +#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
> +
> +#endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-10 6:12 ` Li, Ming4
@ 2024-09-10 7:25 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-10 7:25 UTC (permalink / raw)
To: Li, Ming4, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/10/24 07:12, Li, Ming4 wrote:
> On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Differientiate Type3, aka memory expanders, from Type2, aka device
>> accelerators, with a new function for initializing cxl_dev_state.
>>
>> Create accessors to cxl_dev_state to be used by accel drivers.
>>
>> Add SFC ethernet network driver as the client.
>>
>> Based on https://lore.kernel.org/linux-cxl/168592160379.1948938.12863272903570476312.stgit@dwillia2-xfh.jf.intel.com/
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
>> ---
>> drivers/cxl/core/memdev.c | 52 ++++++++++++++++
>> drivers/cxl/core/pci.c | 1 +
>> drivers/cxl/cxlpci.h | 16 -----
>> drivers/cxl/pci.c | 13 ++--
>> drivers/net/ethernet/sfc/Makefile | 2 +-
>> drivers/net/ethernet/sfc/efx.c | 13 ++++
>> drivers/net/ethernet/sfc/efx_cxl.c | 86 +++++++++++++++++++++++++++
>> drivers/net/ethernet/sfc/efx_cxl.h | 29 +++++++++
>> drivers/net/ethernet/sfc/net_driver.h | 6 ++
>> include/linux/cxl/cxl.h | 21 +++++++
>> include/linux/cxl/pci.h | 23 +++++++
>> 11 files changed, 241 insertions(+), 21 deletions(-)
>> create mode 100644 drivers/net/ethernet/sfc/efx_cxl.c
>> create mode 100644 drivers/net/ethernet/sfc/efx_cxl.h
>> create mode 100644 include/linux/cxl/cxl.h
>> create mode 100644 include/linux/cxl/pci.h
>>
> [...]
>
>>
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> new file mode 100644
>> index 000000000000..e78eefa82123
>> --- /dev/null
>> +++ b/include/linux/cxl/cxl.h
>> @@ -0,0 +1,21 @@
>> +/* SPDX-License-Identifier: GPL-2.0 */
>> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
>> +
>> +#ifndef __CXL_H
>> +#define __CXL_H
>> +
>> +#include <linux/device.h>
>> +
>> +enum cxl_resource {
>> + CXL_ACCEL_RES_DPA,
>> + CXL_ACCEL_RES_RAM,
>> + CXL_ACCEL_RES_PMEM,
>> +};
> Can remove 'ACCEL' from the resource name? they can be used for both type-2 and type-3 devices.
>
Sure.
I'll do it.
Thanks
>> +
>> +struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
>> +
>> +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
>> +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
>> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> + enum cxl_resource);
>> +#endif
>> diff --git a/include/linux/cxl/pci.h b/include/linux/cxl/pci.h
>> new file mode 100644
>> index 000000000000..c337ae8797e6
>> --- /dev/null
>> +++ b/include/linux/cxl/pci.h
>> @@ -0,0 +1,23 @@
>> +/* SPDX-License-Identifier: GPL-2.0 */
>> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
>> +
>> +#ifndef __CXL_ACCEL_PCI_H
>> +#define __CXL_ACCEL_PCI_H
>> +
>> +/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
>> +#define CXL_DVSEC_PCIE_DEVICE 0
>> +#define CXL_DVSEC_CAP_OFFSET 0xA
>> +#define CXL_DVSEC_MEM_CAPABLE BIT(2)
>> +#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
>> +#define CXL_DVSEC_CTRL_OFFSET 0xC
>> +#define CXL_DVSEC_MEM_ENABLE BIT(2)
>> +#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
>> +#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
>> +#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
>> +#define CXL_DVSEC_MEM_ACTIVE BIT(1)
>> +#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
>> +#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
>> +#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
>> +#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
>> +
>> +#endif
>
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-07 8:18 ` [PATCH v3 01/20] cxl: add type2 device basic support alejandro.lucero-palau
2024-09-07 20:26 ` kernel test robot
2024-09-10 6:12 ` Li, Ming4
@ 2024-09-12 8:57 ` Zhi Wang
2024-09-16 9:52 ` Alejandro Lucero Palau
2024-09-12 9:35 ` Zhi Wang
2024-09-13 16:41 ` Jonathan Cameron
4 siblings, 1 reply; 88+ messages in thread
From: Zhi Wang @ 2024-09-12 8:57 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:17 +0100
<alejandro.lucero-palau@amd.com> wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Differientiate Type3, aka memory expanders, from Type2, aka device
> accelerators, with a new function for initializing cxl_dev_state.
>
> Create accessors to cxl_dev_state to be used by accel drivers.
>
> Add SFC ethernet network driver as the client.
>
> Based on
> https://lore.kernel.org/linux-cxl/168592160379.1948938.12863272903570476312.stgit@dwillia2-xfh.jf.intel.com/
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> ---
> drivers/cxl/core/memdev.c | 52 ++++++++++++++++
> drivers/cxl/core/pci.c | 1 +
> drivers/cxl/cxlpci.h | 16 -----
> drivers/cxl/pci.c | 13 ++--
> drivers/net/ethernet/sfc/Makefile | 2 +-
> drivers/net/ethernet/sfc/efx.c | 13 ++++
> drivers/net/ethernet/sfc/efx_cxl.c | 86
> +++++++++++++++++++++++++++ drivers/net/ethernet/sfc/efx_cxl.h |
> 29 +++++++++ drivers/net/ethernet/sfc/net_driver.h | 6 ++
> include/linux/cxl/cxl.h | 21 +++++++
> include/linux/cxl/pci.h | 23 +++++++
> 11 files changed, 241 insertions(+), 21 deletions(-)
> create mode 100644 drivers/net/ethernet/sfc/efx_cxl.c
> create mode 100644 drivers/net/ethernet/sfc/efx_cxl.h
> create mode 100644 include/linux/cxl/cxl.h
> create mode 100644 include/linux/cxl/pci.h
>
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 0277726afd04..10c0a6990f9a 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -1,6 +1,7 @@
> // SPDX-License-Identifier: GPL-2.0-only
> /* Copyright(c) 2020 Intel Corporation. */
>
> +#include <linux/cxl/cxl.h>
> #include <linux/io-64-nonatomic-lo-hi.h>
> #include <linux/firmware.h>
> #include <linux/device.h>
> @@ -615,6 +616,25 @@ static void detach_memdev(struct work_struct
> *work)
> static struct lock_class_key cxl_memdev_key;
>
> +struct cxl_dev_state *cxl_accel_state_create(struct device *dev)
> +{
> + struct cxl_dev_state *cxlds;
> +
> + cxlds = kzalloc(sizeof(*cxlds), GFP_KERNEL);
> + if (!cxlds)
> + return ERR_PTR(-ENOMEM);
> +
> + cxlds->dev = dev;
> + cxlds->type = CXL_DEVTYPE_DEVMEM;
> +
> + cxlds->dpa_res = DEFINE_RES_MEM_NAMED(0, 0, "dpa");
> + cxlds->ram_res = DEFINE_RES_MEM_NAMED(0, 0, "ram");
> + cxlds->pmem_res = DEFINE_RES_MEM_NAMED(0, 0, "pmem");
> +
> + return cxlds;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_accel_state_create, CXL);
> +
> static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state
> *cxlds, const struct file_operations *fops)
> {
> @@ -692,6 +712,38 @@ static int cxl_memdev_open(struct inode *inode,
> struct file *file) return 0;
> }
>
> +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec)
> +{
> + cxlds->cxl_dvsec = dvsec;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_set_dvsec, CXL);
> +
> +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial)
> +{
> + cxlds->serial = serial;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_set_serial, CXL);
> +
> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource
> res,
> + enum cxl_resource type)
> +{
> + switch (type) {
> + case CXL_ACCEL_RES_DPA:
> + cxlds->dpa_res = res;
> + return 0;
> + case CXL_ACCEL_RES_RAM:
> + cxlds->ram_res = res;
> + return 0;
> + case CXL_ACCEL_RES_PMEM:
> + cxlds->pmem_res = res;
> + return 0;
> + default:
> + dev_err(cxlds->dev, "unknown resource type (%u)\n",
> type);
> + return -EINVAL;
> + }
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
> +
> static int cxl_memdev_release_file(struct inode *inode, struct file
> *file) {
> struct cxl_memdev *cxlmd =
> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> index 51132a575b27..3d6564dbda57 100644
> --- a/drivers/cxl/core/pci.c
> +++ b/drivers/cxl/core/pci.c
> @@ -7,6 +7,7 @@
> #include <linux/pci.h>
> #include <linux/pci-doe.h>
> #include <linux/aer.h>
> +#include <linux/cxl/pci.h>
> #include <cxlpci.h>
> #include <cxlmem.h>
> #include <cxl.h>
> diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
> index 4da07727ab9c..eb59019fe5f3 100644
> --- a/drivers/cxl/cxlpci.h
> +++ b/drivers/cxl/cxlpci.h
> @@ -14,22 +14,6 @@
> */
> #define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20)
>
> -/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
> -#define CXL_DVSEC_PCIE_DEVICE
> 0 -#define CXL_DVSEC_CAP_OFFSET 0xA
> -#define CXL_DVSEC_MEM_CAPABLE BIT(2)
> -#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
> -#define CXL_DVSEC_CTRL_OFFSET 0xC
> -#define CXL_DVSEC_MEM_ENABLE BIT(2)
> -#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
> -#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
> -#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
> -#define CXL_DVSEC_MEM_ACTIVE BIT(1)
> -#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
> -#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
> -#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
> -#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
> -
> #define CXL_DVSEC_RANGE_MAX 2
>
> /* CXL 2.0 8.1.4: Non-CXL Function Map DVSEC */
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 4be35dc22202..742a7b2a1be5 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -11,6 +11,8 @@
> #include <linux/pci.h>
> #include <linux/aer.h>
> #include <linux/io.h>
> +#include <linux/cxl/cxl.h>
> +#include <linux/cxl/pci.h>
> #include "cxlmem.h"
> #include "cxlpci.h"
> #include "cxl.h"
> @@ -795,6 +797,7 @@ static int cxl_pci_probe(struct pci_dev *pdev,
> const struct pci_device_id *id) struct cxl_memdev *cxlmd;
> int i, rc, pmu_count;
> bool irq_avail;
> + u16 dvsec;
>
> /*
> * Double check the anonymous union trickery in struct
> cxl_regs @@ -815,12 +818,14 @@ static int cxl_pci_probe(struct
> pci_dev *pdev, const struct pci_device_id *id) pci_set_drvdata(pdev,
> cxlds);
> cxlds->rcd = is_cxl_restricted(pdev);
> - cxlds->serial = pci_get_dsn(pdev);
> - cxlds->cxl_dvsec = pci_find_dvsec_capability(
> - pdev, PCI_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
> - if (!cxlds->cxl_dvsec)
> + cxl_set_serial(cxlds, pci_get_dsn(pdev));
> + dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
> + CXL_DVSEC_PCIE_DEVICE);
> + if (!dvsec)
> dev_warn(&pdev->dev,
> "Device DVSEC not present, skip CXL.mem
> init\n");
> + else
> + cxl_set_dvsec(cxlds, dvsec);
>
> rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
> if (rc)
> diff --git a/drivers/net/ethernet/sfc/Makefile
> b/drivers/net/ethernet/sfc/Makefile index 8f446b9bd5ee..e80c713c3b0c
> 100644 --- a/drivers/net/ethernet/sfc/Makefile
> +++ b/drivers/net/ethernet/sfc/Makefile
> @@ -7,7 +7,7 @@ sfc-y += efx.o efx_common.o
> efx_channels.o nic.o \ mcdi_functions.o mcdi_filters.o mcdi_mon.o \
> ef100.o ef100_nic.o ef100_netdev.o \
> ef100_ethtool.o ef100_rx.o ef100_tx.o \
> - efx_devlink.o
> + efx_devlink.o efx_cxl.o
> sfc-$(CONFIG_SFC_MTD) += mtd.o
> sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o
> ef100_rep.o \ mae.o tc.o tc_bindings.o tc_counters.o \
> diff --git a/drivers/net/ethernet/sfc/efx.c
> b/drivers/net/ethernet/sfc/efx.c index 6f1a01ded7d4..3a7406aa950c
> 100644 --- a/drivers/net/ethernet/sfc/efx.c
> +++ b/drivers/net/ethernet/sfc/efx.c
> @@ -33,6 +33,7 @@
> #include "selftest.h"
> #include "sriov.h"
> #include "efx_devlink.h"
> +#include "efx_cxl.h"
>
> #include "mcdi_port_common.h"
> #include "mcdi_pcol.h"
> @@ -899,6 +900,9 @@ static void efx_pci_remove(struct pci_dev
> *pci_dev) efx_pci_remove_main(efx);
>
> efx_fini_io(efx);
> +
> + efx_cxl_exit(efx);
> +
> pci_dbg(efx->pci_dev, "shutdown successful\n");
>
> efx_fini_devlink_and_unlock(efx);
> @@ -1109,6 +1113,15 @@ static int efx_pci_probe(struct pci_dev
> *pci_dev, if (rc)
> goto fail2;
>
> + /* A successful cxl initialization implies a CXL region
> created to be
> + * used for PIO buffers. If there is no CXL support, or
> initialization
> + * fails, efx_cxl_pio_initialised wll be false and legacy
> PIO buffers
> + * defined at specific PCI BAR regions will be used.
> + */
> + rc = efx_cxl_init(efx);
> + if (rc)
> + pci_err(pci_dev, "CXL initialization failed with
> error %d\n", rc); +
> rc = efx_pci_probe_post_io(efx);
> if (rc) {
> /* On failure, retry once immediately.
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c
> b/drivers/net/ethernet/sfc/efx_cxl.c new file mode 100644
> index 000000000000..bba36cbbab22
> --- /dev/null
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -0,0 +1,86 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/****************************************************************************
> + *
> + * Driver for AMD network controllers and boards
> + * Copyright (C) 2024, Advanced Micro Devices, Inc.
> + *
> + * This program is free software; you can redistribute it and/or
> modify it
> + * under the terms of the GNU General Public License version 2 as
> published
> + * by the Free Software Foundation, incorporated herein by reference.
> + */
> +
> +#include <linux/cxl/cxl.h>
> +#include <linux/cxl/pci.h>
> +#include <linux/pci.h>
> +
> +#include "net_driver.h"
> +#include "efx_cxl.h"
> +
> +#define EFX_CTPIO_BUFFER_SIZE (1024 * 1024 * 256)
> +
Use SZ_256M in include/linux/sizes.h
> +int efx_cxl_init(struct efx_nic *efx)
> +{
> + struct pci_dev *pci_dev = efx->pci_dev;
> + struct efx_cxl *cxl;
> + struct resource res;
> + u16 dvsec;
> + int rc;
> +
> + efx->efx_cxl_pio_initialised = false;
> +
> + dvsec = pci_find_dvsec_capability(pci_dev, PCI_VENDOR_ID_CXL,
> + CXL_DVSEC_PCIE_DEVICE);
> +
> + if (!dvsec)
> + return 0;
> +
> + pci_dbg(pci_dev, "CXL_DVSEC_PCIE_DEVICE capability found\n");
> +
> + efx->cxl = kzalloc(sizeof(*cxl), GFP_KERNEL);
> + if (!efx->cxl)
> + return -ENOMEM;
> +
> + cxl = efx->cxl;
> +
> + cxl->cxlds = cxl_accel_state_create(&pci_dev->dev);
> + if (IS_ERR(cxl->cxlds)) {
> + pci_err(pci_dev, "CXL accel device state failed");
> + kfree(efx->cxl);
> + return -ENOMEM;
> + }
> +
> + cxl_set_dvsec(cxl->cxlds, dvsec);
> + cxl_set_serial(cxl->cxlds, pci_dev->dev.id);
> +
> + res = DEFINE_RES_MEM(0, EFX_CTPIO_BUFFER_SIZE);
> + if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_DPA)) {
> + pci_err(pci_dev, "cxl_set_resource DPA failed\n");
> + rc = -EINVAL;
> + goto err;
> + }
> +
> + res = DEFINE_RES_MEM_NAMED(0, EFX_CTPIO_BUFFER_SIZE, "ram");
> + if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_RAM)) {
> + pci_err(pci_dev, "cxl_set_resource RAM failed\n");
> + rc = -EINVAL;
> + goto err;
> + }
> +
> + return 0;
> +err:
> + kfree(cxl->cxlds);
> + kfree(cxl);
> + efx->cxl = NULL;
> +
> + return rc;
> +}
> +
> +void efx_cxl_exit(struct efx_nic *efx)
> +{
> + if (efx->cxl) {
> + kfree(efx->cxl->cxlds);
> + kfree(efx->cxl);
> + }
> +}
> +
> +MODULE_IMPORT_NS(CXL);
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.h
> b/drivers/net/ethernet/sfc/efx_cxl.h new file mode 100644
> index 000000000000..f57fb2afd124
> --- /dev/null
> +++ b/drivers/net/ethernet/sfc/efx_cxl.h
> @@ -0,0 +1,29 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/****************************************************************************
> + * Driver for AMD network controllers and boards
> + * Copyright (C) 2024, Advanced Micro Devices, Inc.
> + *
> + * This program is free software; you can redistribute it and/or
> modify it
> + * under the terms of the GNU General Public License version 2 as
> published
> + * by the Free Software Foundation, incorporated herein by reference.
> + */
> +
> +#ifndef EFX_CXL_H
> +#define EFX_CXL_H
> +
> +struct efx_nic;
> +struct cxl_dev_state;
> +
> +struct efx_cxl {
> + struct cxl_dev_state *cxlds;
> + struct cxl_memdev *cxlmd;
> + struct cxl_root_decoder *cxlrd;
> + struct cxl_port *endpoint;
> + struct cxl_endpoint_decoder *cxled;
> + struct cxl_region *efx_region;
> + void __iomem *ctpio_cxl;
> +};
> +
> +int efx_cxl_init(struct efx_nic *efx);
> +void efx_cxl_exit(struct efx_nic *efx);
> +#endif
> diff --git a/drivers/net/ethernet/sfc/net_driver.h
> b/drivers/net/ethernet/sfc/net_driver.h index
> b85c51cbe7f9..77261de65e63 100644 ---
> a/drivers/net/ethernet/sfc/net_driver.h +++
> b/drivers/net/ethernet/sfc/net_driver.h @@ -817,6 +817,8 @@ enum
> efx_xdp_tx_queues_mode {
> struct efx_mae;
>
> +struct efx_cxl;
> +
> /**
> * struct efx_nic - an Efx NIC
> * @name: Device name (net device name or bus id before net device
> registered) @@ -963,6 +965,8 @@ struct efx_mae;
> * @tc: state for TC offload (EF100).
> * @devlink: reference to devlink structure owned by this device
> * @dl_port: devlink port associated with the PF
> + * @cxl: details of related cxl objects
> + * @efx_cxl_pio_initialised: clx initialization outcome.
> * @mem_bar: The BAR that is mapped into membase.
> * @reg_base: Offset from the start of the bar to the function
> control window.
> * @monitor_work: Hardware monitor workitem
> @@ -1148,6 +1152,8 @@ struct efx_nic {
>
> struct devlink *devlink;
> struct devlink_port *dl_port;
> + struct efx_cxl *cxl;
> + bool efx_cxl_pio_initialised;
> unsigned int mem_bar;
> u32 reg_base;
>
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> new file mode 100644
> index 000000000000..e78eefa82123
> --- /dev/null
> +++ b/include/linux/cxl/cxl.h
> @@ -0,0 +1,21 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
> +
> +#ifndef __CXL_H
> +#define __CXL_H
> +
> +#include <linux/device.h>
> +
> +enum cxl_resource {
> + CXL_ACCEL_RES_DPA,
> + CXL_ACCEL_RES_RAM,
> + CXL_ACCEL_RES_PMEM,
> +};
> +
> +struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
> +
> +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
> +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource
> res,
> + enum cxl_resource);
> +#endif
> diff --git a/include/linux/cxl/pci.h b/include/linux/cxl/pci.h
> new file mode 100644
> index 000000000000..c337ae8797e6
> --- /dev/null
> +++ b/include/linux/cxl/pci.h
> @@ -0,0 +1,23 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
> +
> +#ifndef __CXL_ACCEL_PCI_H
> +#define __CXL_ACCEL_PCI_H
> +
> +/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
> +#define CXL_DVSEC_PCIE_DEVICE
> 0 +#define CXL_DVSEC_CAP_OFFSET 0xA
> +#define CXL_DVSEC_MEM_CAPABLE BIT(2)
> +#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
> +#define CXL_DVSEC_CTRL_OFFSET 0xC
> +#define CXL_DVSEC_MEM_ENABLE BIT(2)
> +#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
> +#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
> +#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
> +#define CXL_DVSEC_MEM_ACTIVE BIT(1)
> +#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
> +#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
> +#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
> +#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
> +
> +#endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-12 8:57 ` Zhi Wang
@ 2024-09-16 9:52 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 9:52 UTC (permalink / raw)
To: Zhi Wang, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
>> +
>> +#include "net_driver.h"
>> +#include "efx_cxl.h"
>> +
>> +#define EFX_CTPIO_BUFFER_SIZE (1024 * 1024 * 256)
>> +
> Use SZ_256M in include/linux/sizes.h
>
>
I'll do.
Thanks!
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-07 8:18 ` [PATCH v3 01/20] cxl: add type2 device basic support alejandro.lucero-palau
` (2 preceding siblings ...)
2024-09-12 8:57 ` Zhi Wang
@ 2024-09-12 9:35 ` Zhi Wang
2024-09-16 10:03 ` Alejandro Lucero Palau
2024-09-13 16:41 ` Jonathan Cameron
4 siblings, 1 reply; 88+ messages in thread
From: Zhi Wang @ 2024-09-12 9:35 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:17 +0100
<alejandro.lucero-palau@amd.com> wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Differientiate Type3, aka memory expanders, from Type2, aka device
> accelerators, with a new function for initializing cxl_dev_state.
>
> Create accessors to cxl_dev_state to be used by accel drivers.
>
> Add SFC ethernet network driver as the client.
Some thought when working with V2 about the driver initialization
sequence. It seem the initialization sequence for the driver is quite
long (since we do have a lot of stuff to handle) and there has to be a
long error handling coming with it.
Thinking the usual driver initialization sequence is long enough with
error handling sequence, would it be better to lift this efx_cxl_init()
as a initialization wrapper for the driver.
It can take an init params and handling the initialization and errors by
itself according to the params. (The new efx_cxl_init() just
call the wrapper with params)
Thanks,
Zhi.
>
> Based on
> https://lore.kernel.org/linux-cxl/168592160379.1948938.12863272903570476312.stgit@dwillia2-xfh.jf.intel.com/
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> ---
> drivers/cxl/core/memdev.c | 52 ++++++++++++++++
> drivers/cxl/core/pci.c | 1 +
> drivers/cxl/cxlpci.h | 16 -----
> drivers/cxl/pci.c | 13 ++--
> drivers/net/ethernet/sfc/Makefile | 2 +-
> drivers/net/ethernet/sfc/efx.c | 13 ++++
> drivers/net/ethernet/sfc/efx_cxl.c | 86
> +++++++++++++++++++++++++++ drivers/net/ethernet/sfc/efx_cxl.h |
> 29 +++++++++ drivers/net/ethernet/sfc/net_driver.h | 6 ++
> include/linux/cxl/cxl.h | 21 +++++++
> include/linux/cxl/pci.h | 23 +++++++
> 11 files changed, 241 insertions(+), 21 deletions(-)
> create mode 100644 drivers/net/ethernet/sfc/efx_cxl.c
> create mode 100644 drivers/net/ethernet/sfc/efx_cxl.h
> create mode 100644 include/linux/cxl/cxl.h
> create mode 100644 include/linux/cxl/pci.h
>
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 0277726afd04..10c0a6990f9a 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -1,6 +1,7 @@
> // SPDX-License-Identifier: GPL-2.0-only
> /* Copyright(c) 2020 Intel Corporation. */
>
> +#include <linux/cxl/cxl.h>
> #include <linux/io-64-nonatomic-lo-hi.h>
> #include <linux/firmware.h>
> #include <linux/device.h>
> @@ -615,6 +616,25 @@ static void detach_memdev(struct work_struct
> *work)
> static struct lock_class_key cxl_memdev_key;
>
> +struct cxl_dev_state *cxl_accel_state_create(struct device *dev)
> +{
> + struct cxl_dev_state *cxlds;
> +
> + cxlds = kzalloc(sizeof(*cxlds), GFP_KERNEL);
> + if (!cxlds)
> + return ERR_PTR(-ENOMEM);
> +
> + cxlds->dev = dev;
> + cxlds->type = CXL_DEVTYPE_DEVMEM;
> +
> + cxlds->dpa_res = DEFINE_RES_MEM_NAMED(0, 0, "dpa");
> + cxlds->ram_res = DEFINE_RES_MEM_NAMED(0, 0, "ram");
> + cxlds->pmem_res = DEFINE_RES_MEM_NAMED(0, 0, "pmem");
> +
> + return cxlds;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_accel_state_create, CXL);
> +
> static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state
> *cxlds, const struct file_operations *fops)
> {
> @@ -692,6 +712,38 @@ static int cxl_memdev_open(struct inode *inode,
> struct file *file) return 0;
> }
>
> +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec)
> +{
> + cxlds->cxl_dvsec = dvsec;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_set_dvsec, CXL);
> +
> +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial)
> +{
> + cxlds->serial = serial;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_set_serial, CXL);
> +
> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource
> res,
> + enum cxl_resource type)
> +{
> + switch (type) {
> + case CXL_ACCEL_RES_DPA:
> + cxlds->dpa_res = res;
> + return 0;
> + case CXL_ACCEL_RES_RAM:
> + cxlds->ram_res = res;
> + return 0;
> + case CXL_ACCEL_RES_PMEM:
> + cxlds->pmem_res = res;
> + return 0;
> + default:
> + dev_err(cxlds->dev, "unknown resource type (%u)\n",
> type);
> + return -EINVAL;
> + }
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
> +
> static int cxl_memdev_release_file(struct inode *inode, struct file
> *file) {
> struct cxl_memdev *cxlmd =
> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> index 51132a575b27..3d6564dbda57 100644
> --- a/drivers/cxl/core/pci.c
> +++ b/drivers/cxl/core/pci.c
> @@ -7,6 +7,7 @@
> #include <linux/pci.h>
> #include <linux/pci-doe.h>
> #include <linux/aer.h>
> +#include <linux/cxl/pci.h>
> #include <cxlpci.h>
> #include <cxlmem.h>
> #include <cxl.h>
> diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
> index 4da07727ab9c..eb59019fe5f3 100644
> --- a/drivers/cxl/cxlpci.h
> +++ b/drivers/cxl/cxlpci.h
> @@ -14,22 +14,6 @@
> */
> #define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20)
>
> -/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
> -#define CXL_DVSEC_PCIE_DEVICE
> 0 -#define CXL_DVSEC_CAP_OFFSET 0xA
> -#define CXL_DVSEC_MEM_CAPABLE BIT(2)
> -#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
> -#define CXL_DVSEC_CTRL_OFFSET 0xC
> -#define CXL_DVSEC_MEM_ENABLE BIT(2)
> -#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
> -#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
> -#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
> -#define CXL_DVSEC_MEM_ACTIVE BIT(1)
> -#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
> -#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
> -#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
> -#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
> -
> #define CXL_DVSEC_RANGE_MAX 2
>
> /* CXL 2.0 8.1.4: Non-CXL Function Map DVSEC */
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 4be35dc22202..742a7b2a1be5 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -11,6 +11,8 @@
> #include <linux/pci.h>
> #include <linux/aer.h>
> #include <linux/io.h>
> +#include <linux/cxl/cxl.h>
> +#include <linux/cxl/pci.h>
> #include "cxlmem.h"
> #include "cxlpci.h"
> #include "cxl.h"
> @@ -795,6 +797,7 @@ static int cxl_pci_probe(struct pci_dev *pdev,
> const struct pci_device_id *id) struct cxl_memdev *cxlmd;
> int i, rc, pmu_count;
> bool irq_avail;
> + u16 dvsec;
>
> /*
> * Double check the anonymous union trickery in struct
> cxl_regs @@ -815,12 +818,14 @@ static int cxl_pci_probe(struct
> pci_dev *pdev, const struct pci_device_id *id) pci_set_drvdata(pdev,
> cxlds);
> cxlds->rcd = is_cxl_restricted(pdev);
> - cxlds->serial = pci_get_dsn(pdev);
> - cxlds->cxl_dvsec = pci_find_dvsec_capability(
> - pdev, PCI_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
> - if (!cxlds->cxl_dvsec)
> + cxl_set_serial(cxlds, pci_get_dsn(pdev));
> + dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
> + CXL_DVSEC_PCIE_DEVICE);
> + if (!dvsec)
> dev_warn(&pdev->dev,
> "Device DVSEC not present, skip CXL.mem
> init\n");
> + else
> + cxl_set_dvsec(cxlds, dvsec);
>
> rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
> if (rc)
> diff --git a/drivers/net/ethernet/sfc/Makefile
> b/drivers/net/ethernet/sfc/Makefile index 8f446b9bd5ee..e80c713c3b0c
> 100644 --- a/drivers/net/ethernet/sfc/Makefile
> +++ b/drivers/net/ethernet/sfc/Makefile
> @@ -7,7 +7,7 @@ sfc-y += efx.o efx_common.o
> efx_channels.o nic.o \ mcdi_functions.o mcdi_filters.o mcdi_mon.o \
> ef100.o ef100_nic.o ef100_netdev.o \
> ef100_ethtool.o ef100_rx.o ef100_tx.o \
> - efx_devlink.o
> + efx_devlink.o efx_cxl.o
> sfc-$(CONFIG_SFC_MTD) += mtd.o
> sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o
> ef100_rep.o \ mae.o tc.o tc_bindings.o tc_counters.o \
> diff --git a/drivers/net/ethernet/sfc/efx.c
> b/drivers/net/ethernet/sfc/efx.c index 6f1a01ded7d4..3a7406aa950c
> 100644 --- a/drivers/net/ethernet/sfc/efx.c
> +++ b/drivers/net/ethernet/sfc/efx.c
> @@ -33,6 +33,7 @@
> #include "selftest.h"
> #include "sriov.h"
> #include "efx_devlink.h"
> +#include "efx_cxl.h"
>
> #include "mcdi_port_common.h"
> #include "mcdi_pcol.h"
> @@ -899,6 +900,9 @@ static void efx_pci_remove(struct pci_dev
> *pci_dev) efx_pci_remove_main(efx);
>
> efx_fini_io(efx);
> +
> + efx_cxl_exit(efx);
> +
> pci_dbg(efx->pci_dev, "shutdown successful\n");
>
> efx_fini_devlink_and_unlock(efx);
> @@ -1109,6 +1113,15 @@ static int efx_pci_probe(struct pci_dev
> *pci_dev, if (rc)
> goto fail2;
>
> + /* A successful cxl initialization implies a CXL region
> created to be
> + * used for PIO buffers. If there is no CXL support, or
> initialization
> + * fails, efx_cxl_pio_initialised wll be false and legacy
> PIO buffers
> + * defined at specific PCI BAR regions will be used.
> + */
> + rc = efx_cxl_init(efx);
> + if (rc)
> + pci_err(pci_dev, "CXL initialization failed with
> error %d\n", rc); +
> rc = efx_pci_probe_post_io(efx);
> if (rc) {
> /* On failure, retry once immediately.
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c
> b/drivers/net/ethernet/sfc/efx_cxl.c new file mode 100644
> index 000000000000..bba36cbbab22
> --- /dev/null
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -0,0 +1,86 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/****************************************************************************
> + *
> + * Driver for AMD network controllers and boards
> + * Copyright (C) 2024, Advanced Micro Devices, Inc.
> + *
> + * This program is free software; you can redistribute it and/or
> modify it
> + * under the terms of the GNU General Public License version 2 as
> published
> + * by the Free Software Foundation, incorporated herein by reference.
> + */
> +
> +#include <linux/cxl/cxl.h>
> +#include <linux/cxl/pci.h>
> +#include <linux/pci.h>
> +
> +#include "net_driver.h"
> +#include "efx_cxl.h"
> +
> +#define EFX_CTPIO_BUFFER_SIZE (1024 * 1024 * 256)
> +
> +int efx_cxl_init(struct efx_nic *efx)
> +{
> + struct pci_dev *pci_dev = efx->pci_dev;
> + struct efx_cxl *cxl;
> + struct resource res;
> + u16 dvsec;
> + int rc;
> +
> + efx->efx_cxl_pio_initialised = false;
> +
> + dvsec = pci_find_dvsec_capability(pci_dev, PCI_VENDOR_ID_CXL,
> + CXL_DVSEC_PCIE_DEVICE);
> +
> + if (!dvsec)
> + return 0;
> +
> + pci_dbg(pci_dev, "CXL_DVSEC_PCIE_DEVICE capability found\n");
> +
> + efx->cxl = kzalloc(sizeof(*cxl), GFP_KERNEL);
> + if (!efx->cxl)
> + return -ENOMEM;
> +
> + cxl = efx->cxl;
> +
> + cxl->cxlds = cxl_accel_state_create(&pci_dev->dev);
> + if (IS_ERR(cxl->cxlds)) {
> + pci_err(pci_dev, "CXL accel device state failed");
> + kfree(efx->cxl);
> + return -ENOMEM;
> + }
> +
> + cxl_set_dvsec(cxl->cxlds, dvsec);
> + cxl_set_serial(cxl->cxlds, pci_dev->dev.id);
> +
> + res = DEFINE_RES_MEM(0, EFX_CTPIO_BUFFER_SIZE);
> + if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_DPA)) {
> + pci_err(pci_dev, "cxl_set_resource DPA failed\n");
> + rc = -EINVAL;
> + goto err;
> + }
> +
> + res = DEFINE_RES_MEM_NAMED(0, EFX_CTPIO_BUFFER_SIZE, "ram");
> + if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_RAM)) {
> + pci_err(pci_dev, "cxl_set_resource RAM failed\n");
> + rc = -EINVAL;
> + goto err;
> + }
> +
> + return 0;
> +err:
> + kfree(cxl->cxlds);
> + kfree(cxl);
> + efx->cxl = NULL;
> +
> + return rc;
> +}
> +
> +void efx_cxl_exit(struct efx_nic *efx)
> +{
> + if (efx->cxl) {
> + kfree(efx->cxl->cxlds);
> + kfree(efx->cxl);
> + }
> +}
> +
> +MODULE_IMPORT_NS(CXL);
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.h
> b/drivers/net/ethernet/sfc/efx_cxl.h new file mode 100644
> index 000000000000..f57fb2afd124
> --- /dev/null
> +++ b/drivers/net/ethernet/sfc/efx_cxl.h
> @@ -0,0 +1,29 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/****************************************************************************
> + * Driver for AMD network controllers and boards
> + * Copyright (C) 2024, Advanced Micro Devices, Inc.
> + *
> + * This program is free software; you can redistribute it and/or
> modify it
> + * under the terms of the GNU General Public License version 2 as
> published
> + * by the Free Software Foundation, incorporated herein by reference.
> + */
> +
> +#ifndef EFX_CXL_H
> +#define EFX_CXL_H
> +
> +struct efx_nic;
> +struct cxl_dev_state;
> +
> +struct efx_cxl {
> + struct cxl_dev_state *cxlds;
> + struct cxl_memdev *cxlmd;
> + struct cxl_root_decoder *cxlrd;
> + struct cxl_port *endpoint;
> + struct cxl_endpoint_decoder *cxled;
> + struct cxl_region *efx_region;
> + void __iomem *ctpio_cxl;
> +};
> +
> +int efx_cxl_init(struct efx_nic *efx);
> +void efx_cxl_exit(struct efx_nic *efx);
> +#endif
> diff --git a/drivers/net/ethernet/sfc/net_driver.h
> b/drivers/net/ethernet/sfc/net_driver.h index
> b85c51cbe7f9..77261de65e63 100644 ---
> a/drivers/net/ethernet/sfc/net_driver.h +++
> b/drivers/net/ethernet/sfc/net_driver.h @@ -817,6 +817,8 @@ enum
> efx_xdp_tx_queues_mode {
> struct efx_mae;
>
> +struct efx_cxl;
> +
> /**
> * struct efx_nic - an Efx NIC
> * @name: Device name (net device name or bus id before net device
> registered) @@ -963,6 +965,8 @@ struct efx_mae;
> * @tc: state for TC offload (EF100).
> * @devlink: reference to devlink structure owned by this device
> * @dl_port: devlink port associated with the PF
> + * @cxl: details of related cxl objects
> + * @efx_cxl_pio_initialised: clx initialization outcome.
> * @mem_bar: The BAR that is mapped into membase.
> * @reg_base: Offset from the start of the bar to the function
> control window.
> * @monitor_work: Hardware monitor workitem
> @@ -1148,6 +1152,8 @@ struct efx_nic {
>
> struct devlink *devlink;
> struct devlink_port *dl_port;
> + struct efx_cxl *cxl;
> + bool efx_cxl_pio_initialised;
> unsigned int mem_bar;
> u32 reg_base;
>
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> new file mode 100644
> index 000000000000..e78eefa82123
> --- /dev/null
> +++ b/include/linux/cxl/cxl.h
> @@ -0,0 +1,21 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
> +
> +#ifndef __CXL_H
> +#define __CXL_H
> +
> +#include <linux/device.h>
> +
> +enum cxl_resource {
> + CXL_ACCEL_RES_DPA,
> + CXL_ACCEL_RES_RAM,
> + CXL_ACCEL_RES_PMEM,
> +};
> +
> +struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
> +
> +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
> +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource
> res,
> + enum cxl_resource);
> +#endif
> diff --git a/include/linux/cxl/pci.h b/include/linux/cxl/pci.h
> new file mode 100644
> index 000000000000..c337ae8797e6
> --- /dev/null
> +++ b/include/linux/cxl/pci.h
> @@ -0,0 +1,23 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
> +
> +#ifndef __CXL_ACCEL_PCI_H
> +#define __CXL_ACCEL_PCI_H
> +
> +/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
> +#define CXL_DVSEC_PCIE_DEVICE
> 0 +#define CXL_DVSEC_CAP_OFFSET 0xA
> +#define CXL_DVSEC_MEM_CAPABLE BIT(2)
> +#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
> +#define CXL_DVSEC_CTRL_OFFSET 0xC
> +#define CXL_DVSEC_MEM_ENABLE BIT(2)
> +#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
> +#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
> +#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
> +#define CXL_DVSEC_MEM_ACTIVE BIT(1)
> +#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
> +#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
> +#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
> +#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
> +
> +#endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-12 9:35 ` Zhi Wang
@ 2024-09-16 10:03 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 10:03 UTC (permalink / raw)
To: Zhi Wang, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/12/24 10:35, Zhi Wang wrote:
> On Sat, 7 Sep 2024 09:18:17 +0100
> <alejandro.lucero-palau@amd.com> wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Differientiate Type3, aka memory expanders, from Type2, aka device
>> accelerators, with a new function for initializing cxl_dev_state.
>>
>> Create accessors to cxl_dev_state to be used by accel drivers.
>>
>> Add SFC ethernet network driver as the client.
> Some thought when working with V2 about the driver initialization
> sequence. It seem the initialization sequence for the driver is quite
> long (since we do have a lot of stuff to handle) and there has to be a
> long error handling coming with it.
>
> Thinking the usual driver initialization sequence is long enough with
> error handling sequence, would it be better to lift this efx_cxl_init()
> as a initialization wrapper for the driver.
>
> It can take an init params and handling the initialization and errors by
> itself according to the params. (The new efx_cxl_init() just
> call the wrapper with params)
Sorry, I do not follow your comment.
If you are suggesting to efx_cxl_init being the main init function for
sfc, I'm afraid that is not easy to happen and I do not think it is
justified.
Thanks
> Thanks,
> Zhi.
>> Based on
>> https://lore.kernel.org/linux-cxl/168592160379.1948938.12863272903570476312.stgit@dwillia2-xfh.jf.intel.com/
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
>> ---
>> drivers/cxl/core/memdev.c | 52 ++++++++++++++++
>> drivers/cxl/core/pci.c | 1 +
>> drivers/cxl/cxlpci.h | 16 -----
>> drivers/cxl/pci.c | 13 ++--
>> drivers/net/ethernet/sfc/Makefile | 2 +-
>> drivers/net/ethernet/sfc/efx.c | 13 ++++
>> drivers/net/ethernet/sfc/efx_cxl.c | 86
>> +++++++++++++++++++++++++++ drivers/net/ethernet/sfc/efx_cxl.h |
>> 29 +++++++++ drivers/net/ethernet/sfc/net_driver.h | 6 ++
>> include/linux/cxl/cxl.h | 21 +++++++
>> include/linux/cxl/pci.h | 23 +++++++
>> 11 files changed, 241 insertions(+), 21 deletions(-)
>> create mode 100644 drivers/net/ethernet/sfc/efx_cxl.c
>> create mode 100644 drivers/net/ethernet/sfc/efx_cxl.h
>> create mode 100644 include/linux/cxl/cxl.h
>> create mode 100644 include/linux/cxl/pci.h
>>
>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>> index 0277726afd04..10c0a6990f9a 100644
>> --- a/drivers/cxl/core/memdev.c
>> +++ b/drivers/cxl/core/memdev.c
>> @@ -1,6 +1,7 @@
>> // SPDX-License-Identifier: GPL-2.0-only
>> /* Copyright(c) 2020 Intel Corporation. */
>>
>> +#include <linux/cxl/cxl.h>
>> #include <linux/io-64-nonatomic-lo-hi.h>
>> #include <linux/firmware.h>
>> #include <linux/device.h>
>> @@ -615,6 +616,25 @@ static void detach_memdev(struct work_struct
>> *work)
>> static struct lock_class_key cxl_memdev_key;
>>
>> +struct cxl_dev_state *cxl_accel_state_create(struct device *dev)
>> +{
>> + struct cxl_dev_state *cxlds;
>> +
>> + cxlds = kzalloc(sizeof(*cxlds), GFP_KERNEL);
>> + if (!cxlds)
>> + return ERR_PTR(-ENOMEM);
>> +
>> + cxlds->dev = dev;
>> + cxlds->type = CXL_DEVTYPE_DEVMEM;
>> +
>> + cxlds->dpa_res = DEFINE_RES_MEM_NAMED(0, 0, "dpa");
>> + cxlds->ram_res = DEFINE_RES_MEM_NAMED(0, 0, "ram");
>> + cxlds->pmem_res = DEFINE_RES_MEM_NAMED(0, 0, "pmem");
>> +
>> + return cxlds;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_accel_state_create, CXL);
>> +
>> static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state
>> *cxlds, const struct file_operations *fops)
>> {
>> @@ -692,6 +712,38 @@ static int cxl_memdev_open(struct inode *inode,
>> struct file *file) return 0;
>> }
>>
>> +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec)
>> +{
>> + cxlds->cxl_dvsec = dvsec;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_set_dvsec, CXL);
>> +
>> +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial)
>> +{
>> + cxlds->serial = serial;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_set_serial, CXL);
>> +
>> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource
>> res,
>> + enum cxl_resource type)
>> +{
>> + switch (type) {
>> + case CXL_ACCEL_RES_DPA:
>> + cxlds->dpa_res = res;
>> + return 0;
>> + case CXL_ACCEL_RES_RAM:
>> + cxlds->ram_res = res;
>> + return 0;
>> + case CXL_ACCEL_RES_PMEM:
>> + cxlds->pmem_res = res;
>> + return 0;
>> + default:
>> + dev_err(cxlds->dev, "unknown resource type (%u)\n",
>> type);
>> + return -EINVAL;
>> + }
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
>> +
>> static int cxl_memdev_release_file(struct inode *inode, struct file
>> *file) {
>> struct cxl_memdev *cxlmd =
>> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
>> index 51132a575b27..3d6564dbda57 100644
>> --- a/drivers/cxl/core/pci.c
>> +++ b/drivers/cxl/core/pci.c
>> @@ -7,6 +7,7 @@
>> #include <linux/pci.h>
>> #include <linux/pci-doe.h>
>> #include <linux/aer.h>
>> +#include <linux/cxl/pci.h>
>> #include <cxlpci.h>
>> #include <cxlmem.h>
>> #include <cxl.h>
>> diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
>> index 4da07727ab9c..eb59019fe5f3 100644
>> --- a/drivers/cxl/cxlpci.h
>> +++ b/drivers/cxl/cxlpci.h
>> @@ -14,22 +14,6 @@
>> */
>> #define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20)
>>
>> -/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
>> -#define CXL_DVSEC_PCIE_DEVICE
>> 0 -#define CXL_DVSEC_CAP_OFFSET 0xA
>> -#define CXL_DVSEC_MEM_CAPABLE BIT(2)
>> -#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
>> -#define CXL_DVSEC_CTRL_OFFSET 0xC
>> -#define CXL_DVSEC_MEM_ENABLE BIT(2)
>> -#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
>> -#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
>> -#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
>> -#define CXL_DVSEC_MEM_ACTIVE BIT(1)
>> -#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
>> -#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
>> -#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
>> -#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
>> -
>> #define CXL_DVSEC_RANGE_MAX 2
>>
>> /* CXL 2.0 8.1.4: Non-CXL Function Map DVSEC */
>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>> index 4be35dc22202..742a7b2a1be5 100644
>> --- a/drivers/cxl/pci.c
>> +++ b/drivers/cxl/pci.c
>> @@ -11,6 +11,8 @@
>> #include <linux/pci.h>
>> #include <linux/aer.h>
>> #include <linux/io.h>
>> +#include <linux/cxl/cxl.h>
>> +#include <linux/cxl/pci.h>
>> #include "cxlmem.h"
>> #include "cxlpci.h"
>> #include "cxl.h"
>> @@ -795,6 +797,7 @@ static int cxl_pci_probe(struct pci_dev *pdev,
>> const struct pci_device_id *id) struct cxl_memdev *cxlmd;
>> int i, rc, pmu_count;
>> bool irq_avail;
>> + u16 dvsec;
>>
>> /*
>> * Double check the anonymous union trickery in struct
>> cxl_regs @@ -815,12 +818,14 @@ static int cxl_pci_probe(struct
>> pci_dev *pdev, const struct pci_device_id *id) pci_set_drvdata(pdev,
>> cxlds);
>> cxlds->rcd = is_cxl_restricted(pdev);
>> - cxlds->serial = pci_get_dsn(pdev);
>> - cxlds->cxl_dvsec = pci_find_dvsec_capability(
>> - pdev, PCI_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
>> - if (!cxlds->cxl_dvsec)
>> + cxl_set_serial(cxlds, pci_get_dsn(pdev));
>> + dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
>> + CXL_DVSEC_PCIE_DEVICE);
>> + if (!dvsec)
>> dev_warn(&pdev->dev,
>> "Device DVSEC not present, skip CXL.mem
>> init\n");
>> + else
>> + cxl_set_dvsec(cxlds, dvsec);
>>
>> rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
>> if (rc)
>> diff --git a/drivers/net/ethernet/sfc/Makefile
>> b/drivers/net/ethernet/sfc/Makefile index 8f446b9bd5ee..e80c713c3b0c
>> 100644 --- a/drivers/net/ethernet/sfc/Makefile
>> +++ b/drivers/net/ethernet/sfc/Makefile
>> @@ -7,7 +7,7 @@ sfc-y += efx.o efx_common.o
>> efx_channels.o nic.o \ mcdi_functions.o mcdi_filters.o mcdi_mon.o \
>> ef100.o ef100_nic.o ef100_netdev.o \
>> ef100_ethtool.o ef100_rx.o ef100_tx.o \
>> - efx_devlink.o
>> + efx_devlink.o efx_cxl.o
>> sfc-$(CONFIG_SFC_MTD) += mtd.o
>> sfc-$(CONFIG_SFC_SRIOV) += sriov.o ef10_sriov.o ef100_sriov.o
>> ef100_rep.o \ mae.o tc.o tc_bindings.o tc_counters.o \
>> diff --git a/drivers/net/ethernet/sfc/efx.c
>> b/drivers/net/ethernet/sfc/efx.c index 6f1a01ded7d4..3a7406aa950c
>> 100644 --- a/drivers/net/ethernet/sfc/efx.c
>> +++ b/drivers/net/ethernet/sfc/efx.c
>> @@ -33,6 +33,7 @@
>> #include "selftest.h"
>> #include "sriov.h"
>> #include "efx_devlink.h"
>> +#include "efx_cxl.h"
>>
>> #include "mcdi_port_common.h"
>> #include "mcdi_pcol.h"
>> @@ -899,6 +900,9 @@ static void efx_pci_remove(struct pci_dev
>> *pci_dev) efx_pci_remove_main(efx);
>>
>> efx_fini_io(efx);
>> +
>> + efx_cxl_exit(efx);
>> +
>> pci_dbg(efx->pci_dev, "shutdown successful\n");
>>
>> efx_fini_devlink_and_unlock(efx);
>> @@ -1109,6 +1113,15 @@ static int efx_pci_probe(struct pci_dev
>> *pci_dev, if (rc)
>> goto fail2;
>>
>> + /* A successful cxl initialization implies a CXL region
>> created to be
>> + * used for PIO buffers. If there is no CXL support, or
>> initialization
>> + * fails, efx_cxl_pio_initialised wll be false and legacy
>> PIO buffers
>> + * defined at specific PCI BAR regions will be used.
>> + */
>> + rc = efx_cxl_init(efx);
>> + if (rc)
>> + pci_err(pci_dev, "CXL initialization failed with
>> error %d\n", rc); +
>> rc = efx_pci_probe_post_io(efx);
>> if (rc) {
>> /* On failure, retry once immediately.
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c
>> b/drivers/net/ethernet/sfc/efx_cxl.c new file mode 100644
>> index 000000000000..bba36cbbab22
>> --- /dev/null
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -0,0 +1,86 @@
>> +// SPDX-License-Identifier: GPL-2.0-only
>> +/****************************************************************************
>> + *
>> + * Driver for AMD network controllers and boards
>> + * Copyright (C) 2024, Advanced Micro Devices, Inc.
>> + *
>> + * This program is free software; you can redistribute it and/or
>> modify it
>> + * under the terms of the GNU General Public License version 2 as
>> published
>> + * by the Free Software Foundation, incorporated herein by reference.
>> + */
>> +
>> +#include <linux/cxl/cxl.h>
>> +#include <linux/cxl/pci.h>
>> +#include <linux/pci.h>
>> +
>> +#include "net_driver.h"
>> +#include "efx_cxl.h"
>> +
>> +#define EFX_CTPIO_BUFFER_SIZE (1024 * 1024 * 256)
>> +
>> +int efx_cxl_init(struct efx_nic *efx)
>> +{
>> + struct pci_dev *pci_dev = efx->pci_dev;
>> + struct efx_cxl *cxl;
>> + struct resource res;
>> + u16 dvsec;
>> + int rc;
>> +
>> + efx->efx_cxl_pio_initialised = false;
>> +
>> + dvsec = pci_find_dvsec_capability(pci_dev, PCI_VENDOR_ID_CXL,
>> + CXL_DVSEC_PCIE_DEVICE);
>> +
>> + if (!dvsec)
>> + return 0;
>> +
>> + pci_dbg(pci_dev, "CXL_DVSEC_PCIE_DEVICE capability found\n");
>> +
>> + efx->cxl = kzalloc(sizeof(*cxl), GFP_KERNEL);
>> + if (!efx->cxl)
>> + return -ENOMEM;
>> +
>> + cxl = efx->cxl;
>> +
>> + cxl->cxlds = cxl_accel_state_create(&pci_dev->dev);
>> + if (IS_ERR(cxl->cxlds)) {
>> + pci_err(pci_dev, "CXL accel device state failed");
>> + kfree(efx->cxl);
>> + return -ENOMEM;
>> + }
>> +
>> + cxl_set_dvsec(cxl->cxlds, dvsec);
>> + cxl_set_serial(cxl->cxlds, pci_dev->dev.id);
>> +
>> + res = DEFINE_RES_MEM(0, EFX_CTPIO_BUFFER_SIZE);
>> + if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_DPA)) {
>> + pci_err(pci_dev, "cxl_set_resource DPA failed\n");
>> + rc = -EINVAL;
>> + goto err;
>> + }
>> +
>> + res = DEFINE_RES_MEM_NAMED(0, EFX_CTPIO_BUFFER_SIZE, "ram");
>> + if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_RAM)) {
>> + pci_err(pci_dev, "cxl_set_resource RAM failed\n");
>> + rc = -EINVAL;
>> + goto err;
>> + }
>> +
>> + return 0;
>> +err:
>> + kfree(cxl->cxlds);
>> + kfree(cxl);
>> + efx->cxl = NULL;
>> +
>> + return rc;
>> +}
>> +
>> +void efx_cxl_exit(struct efx_nic *efx)
>> +{
>> + if (efx->cxl) {
>> + kfree(efx->cxl->cxlds);
>> + kfree(efx->cxl);
>> + }
>> +}
>> +
>> +MODULE_IMPORT_NS(CXL);
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.h
>> b/drivers/net/ethernet/sfc/efx_cxl.h new file mode 100644
>> index 000000000000..f57fb2afd124
>> --- /dev/null
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.h
>> @@ -0,0 +1,29 @@
>> +/* SPDX-License-Identifier: GPL-2.0-only */
>> +/****************************************************************************
>> + * Driver for AMD network controllers and boards
>> + * Copyright (C) 2024, Advanced Micro Devices, Inc.
>> + *
>> + * This program is free software; you can redistribute it and/or
>> modify it
>> + * under the terms of the GNU General Public License version 2 as
>> published
>> + * by the Free Software Foundation, incorporated herein by reference.
>> + */
>> +
>> +#ifndef EFX_CXL_H
>> +#define EFX_CXL_H
>> +
>> +struct efx_nic;
>> +struct cxl_dev_state;
>> +
>> +struct efx_cxl {
>> + struct cxl_dev_state *cxlds;
>> + struct cxl_memdev *cxlmd;
>> + struct cxl_root_decoder *cxlrd;
>> + struct cxl_port *endpoint;
>> + struct cxl_endpoint_decoder *cxled;
>> + struct cxl_region *efx_region;
>> + void __iomem *ctpio_cxl;
>> +};
>> +
>> +int efx_cxl_init(struct efx_nic *efx);
>> +void efx_cxl_exit(struct efx_nic *efx);
>> +#endif
>> diff --git a/drivers/net/ethernet/sfc/net_driver.h
>> b/drivers/net/ethernet/sfc/net_driver.h index
>> b85c51cbe7f9..77261de65e63 100644 ---
>> a/drivers/net/ethernet/sfc/net_driver.h +++
>> b/drivers/net/ethernet/sfc/net_driver.h @@ -817,6 +817,8 @@ enum
>> efx_xdp_tx_queues_mode {
>> struct efx_mae;
>>
>> +struct efx_cxl;
>> +
>> /**
>> * struct efx_nic - an Efx NIC
>> * @name: Device name (net device name or bus id before net device
>> registered) @@ -963,6 +965,8 @@ struct efx_mae;
>> * @tc: state for TC offload (EF100).
>> * @devlink: reference to devlink structure owned by this device
>> * @dl_port: devlink port associated with the PF
>> + * @cxl: details of related cxl objects
>> + * @efx_cxl_pio_initialised: clx initialization outcome.
>> * @mem_bar: The BAR that is mapped into membase.
>> * @reg_base: Offset from the start of the bar to the function
>> control window.
>> * @monitor_work: Hardware monitor workitem
>> @@ -1148,6 +1152,8 @@ struct efx_nic {
>>
>> struct devlink *devlink;
>> struct devlink_port *dl_port;
>> + struct efx_cxl *cxl;
>> + bool efx_cxl_pio_initialised;
>> unsigned int mem_bar;
>> u32 reg_base;
>>
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> new file mode 100644
>> index 000000000000..e78eefa82123
>> --- /dev/null
>> +++ b/include/linux/cxl/cxl.h
>> @@ -0,0 +1,21 @@
>> +/* SPDX-License-Identifier: GPL-2.0 */
>> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
>> +
>> +#ifndef __CXL_H
>> +#define __CXL_H
>> +
>> +#include <linux/device.h>
>> +
>> +enum cxl_resource {
>> + CXL_ACCEL_RES_DPA,
>> + CXL_ACCEL_RES_RAM,
>> + CXL_ACCEL_RES_PMEM,
>> +};
>> +
>> +struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
>> +
>> +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
>> +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
>> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource
>> res,
>> + enum cxl_resource);
>> +#endif
>> diff --git a/include/linux/cxl/pci.h b/include/linux/cxl/pci.h
>> new file mode 100644
>> index 000000000000..c337ae8797e6
>> --- /dev/null
>> +++ b/include/linux/cxl/pci.h
>> @@ -0,0 +1,23 @@
>> +/* SPDX-License-Identifier: GPL-2.0 */
>> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
>> +
>> +#ifndef __CXL_ACCEL_PCI_H
>> +#define __CXL_ACCEL_PCI_H
>> +
>> +/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
>> +#define CXL_DVSEC_PCIE_DEVICE
>> 0 +#define CXL_DVSEC_CAP_OFFSET 0xA
>> +#define CXL_DVSEC_MEM_CAPABLE BIT(2)
>> +#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
>> +#define CXL_DVSEC_CTRL_OFFSET 0xC
>> +#define CXL_DVSEC_MEM_ENABLE BIT(2)
>> +#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
>> +#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
>> +#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
>> +#define CXL_DVSEC_MEM_ACTIVE BIT(1)
>> +#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
>> +#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
>> +#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
>> +#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
>> +
>> +#endif
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-07 8:18 ` [PATCH v3 01/20] cxl: add type2 device basic support alejandro.lucero-palau
` (3 preceding siblings ...)
2024-09-12 9:35 ` Zhi Wang
@ 2024-09-13 16:41 ` Jonathan Cameron
2024-09-16 12:03 ` Alejandro Lucero Palau
4 siblings, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 16:41 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:17 +0100
<alejandro.lucero-palau@amd.com> wrote:
> From: Alejandro Lucero <alucerop@amd.com>
Hi Alejandro,
I'm mainly looking at these to get my head back into this support
for discussions next week but will probably leave
lots of trivial review feedback as I go.
And to advertise that:
https://lpc.events/event/18/contributions/1828/
>
> Differientiate Type3, aka memory expanders, from Type2, aka device
Spell check. Differentiate.
> accelerators, with a new function for initializing cxl_dev_state.
>
> Create accessors to cxl_dev_state to be used by accel drivers.
>
> Add SFC ethernet network driver as the client.
Minor thing (And others may disagree) but I'd split this to be nice
to others who might want to backport the type2 support but not
the sfc changes (as they are supporting some other hardware).
>
> Based on https://lore.kernel.org/linux-cxl/168592160379.1948938.12863272903570476312.stgit@dwillia2-xfh.jf.intel.com/
Maybe make that a link tag Link: .... # [1]
and have
Based on [1] here.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> + enum cxl_resource type)
> +{
> + switch (type) {
> + case CXL_ACCEL_RES_DPA:
> + cxlds->dpa_res = res;
> + return 0;
> + case CXL_ACCEL_RES_RAM:
> + cxlds->ram_res = res;
> + return 0;
> + case CXL_ACCEL_RES_PMEM:
> + cxlds->pmem_res = res;
> + return 0;
> + default:
> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
It's an enum, do we need the default? Hence do we need the return value?
> + return -EINVAL;
> + }
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
> +
> static int cxl_memdev_release_file(struct inode *inode, struct file *file)
> {
> struct cxl_memdev *cxlmd =
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 4be35dc22202..742a7b2a1be5 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -11,6 +11,8 @@
> #include <linux/pci.h>
> #include <linux/aer.h>
> #include <linux/io.h>
> +#include <linux/cxl/cxl.h>
> +#include <linux/cxl/pci.h>
> #include "cxlmem.h"
> #include "cxlpci.h"
> #include "cxl.h"
> @@ -795,6 +797,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> struct cxl_memdev *cxlmd;
> int i, rc, pmu_count;
> bool irq_avail;
> + u16 dvsec;
>
> /*
> * Double check the anonymous union trickery in struct cxl_regs
> @@ -815,12 +818,14 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> pci_set_drvdata(pdev, cxlds);
>
> cxlds->rcd = is_cxl_restricted(pdev);
> - cxlds->serial = pci_get_dsn(pdev);
> - cxlds->cxl_dvsec = pci_find_dvsec_capability(
> - pdev, PCI_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
> - if (!cxlds->cxl_dvsec)
> + cxl_set_serial(cxlds, pci_get_dsn(pdev));
> + dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
> + CXL_DVSEC_PCIE_DEVICE);
> + if (!dvsec)
> dev_warn(&pdev->dev,
> "Device DVSEC not present, skip CXL.mem init\n");
> + else
> + cxl_set_dvsec(cxlds, dvsec);
Set it unconditionally perhaps. If it's NULL that's fine and then it corresponds
directly to the previous
>
> rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
> if (rc)
> diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
> index 6f1a01ded7d4..3a7406aa950c 100644
> --- a/drivers/net/ethernet/sfc/efx.c
> +++ b/drivers/net/ethernet/sfc/efx.c
> @@ -33,6 +33,7 @@
> #include "selftest.h"
> #include "sriov.h"
> #include "efx_devlink.h"
> +#include "efx_cxl.h"
>
> #include "mcdi_port_common.h"
> #include "mcdi_pcol.h"
> @@ -899,6 +900,9 @@ static void efx_pci_remove(struct pci_dev *pci_dev)
> efx_pci_remove_main(efx);
>
> efx_fini_io(efx);
> +
> + efx_cxl_exit(efx);
> +
> pci_dbg(efx->pci_dev, "shutdown successful\n");
>
> efx_fini_devlink_and_unlock(efx);
> @@ -1109,6 +1113,15 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
> if (rc)
> goto fail2;
>
> + /* A successful cxl initialization implies a CXL region created to be
> + * used for PIO buffers. If there is no CXL support, or initialization
> + * fails, efx_cxl_pio_initialised wll be false and legacy PIO buffers
> + * defined at specific PCI BAR regions will be used.
> + */
> + rc = efx_cxl_init(efx);
> + if (rc)
> + pci_err(pci_dev, "CXL initialization failed with error %d\n", rc);
If you are carrying on anyway is pci_info() more appropriate?
Personally I dislike muddling on in error cases, but understand
it can be useful on occasion at the cost of more complex flows.
> +
> rc = efx_pci_probe_post_io(efx);
> if (rc) {
> /* On failure, retry once immediately.
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> new file mode 100644
> index 000000000000..bba36cbbab22
> --- /dev/null
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -0,0 +1,86 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/****************************************************************************
> + *
> + * Driver for AMD network controllers and boards
> + * Copyright (C) 2024, Advanced Micro Devices, Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU General Public License version 2 as published
> + * by the Free Software Foundation, incorporated herein by reference.
> + */
> +
> +#include <linux/cxl/cxl.h>
> +#include <linux/cxl/pci.h>
> +#include <linux/pci.h>
> +
> +#include "net_driver.h"
> +#include "efx_cxl.h"
> +
> +#define EFX_CTPIO_BUFFER_SIZE (1024 * 1024 * 256)
> +
> +int efx_cxl_init(struct efx_nic *efx)
> +{
> + struct pci_dev *pci_dev = efx->pci_dev;
> + struct efx_cxl *cxl;
> + struct resource res;
> + u16 dvsec;
> + int rc;
> +
> + efx->efx_cxl_pio_initialised = false;
> +
> + dvsec = pci_find_dvsec_capability(pci_dev, PCI_VENDOR_ID_CXL,
> + CXL_DVSEC_PCIE_DEVICE);
> +
Trivial but probably no blank line here. Keeps the error condition tightly
grouped with the call.
> + if (!dvsec)
> + return 0;
> +
> + pci_dbg(pci_dev, "CXL_DVSEC_PCIE_DEVICE capability found\n");
> +
> + efx->cxl = kzalloc(sizeof(*cxl), GFP_KERNEL);
> + if (!efx->cxl)
> + return -ENOMEM;
> +
> + cxl = efx->cxl;
Rather than setting it back to zero in some error paths I'd
suggest keeping it as local only until you know everything
succeeded.
cxl = kzalloc(...)
//maybe also cxlds as then you can use __free() to handle the
//cleanup paths for both allowing early returns instead
//of gotos.
...
efx->cxl = cxl;
return 0;
> +
> + cxl->cxlds = cxl_accel_state_create(&pci_dev->dev);
> + if (IS_ERR(cxl->cxlds)) {
> + pci_err(pci_dev, "CXL accel device state failed");
> + kfree(efx->cxl);
Use the a separate label below. Error blocks in a given function
should probably do one or the other between going to labels
or handling locally. Mixture is harder to read.
> + return -ENOMEM;
> + }
> +
> + cxl_set_dvsec(cxl->cxlds, dvsec);
> + cxl_set_serial(cxl->cxlds, pci_dev->dev.id);
> +
> + res = DEFINE_RES_MEM(0, EFX_CTPIO_BUFFER_SIZE);
> + if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_DPA)) {
> + pci_err(pci_dev, "cxl_set_resource DPA failed\n");
> + rc = -EINVAL;
> + goto err;
> + }
> +
> + res = DEFINE_RES_MEM_NAMED(0, EFX_CTPIO_BUFFER_SIZE, "ram");
> + if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_RAM)) {
> + pci_err(pci_dev, "cxl_set_resource RAM failed\n");
> + rc = -EINVAL;
> + goto err;
> + }
> +
> + return 0;
> +err:
> + kfree(cxl->cxlds);
> + kfree(cxl);
> + efx->cxl = NULL;
> +
> + return rc;
> +}
> +
> +void efx_cxl_exit(struct efx_nic *efx)
> +{
> + if (efx->cxl) {
> + kfree(efx->cxl->cxlds);
> + kfree(efx->cxl);
> + }
> +}
> +
> +MODULE_IMPORT_NS(CXL);
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.h b/drivers/net/ethernet/sfc/efx_cxl.h
> new file mode 100644
> index 000000000000..f57fb2afd124
> --- /dev/null
> +++ b/drivers/net/ethernet/sfc/efx_cxl.h
...
> +struct efx_cxl {
> + struct cxl_dev_state *cxlds;
> + struct cxl_memdev *cxlmd;
> + struct cxl_root_decoder *cxlrd;
> + struct cxl_port *endpoint;
> + struct cxl_endpoint_decoder *cxled;
> + struct cxl_region *efx_region;
Why is the region efx_ prefixed but nothing else?
Feels a little random.
> + void __iomem *ctpio_cxl;
> +};
> +
> +int efx_cxl_init(struct efx_nic *efx);
> +void efx_cxl_exit(struct efx_nic *efx);
> +#endif
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> new file mode 100644
> index 000000000000..e78eefa82123
> --- /dev/null
> +++ b/include/linux/cxl/cxl.h
> @@ -0,0 +1,21 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
> +
> +#ifndef __CXL_H
> +#define __CXL_H
> +
> +#include <linux/device.h>
> +
> +enum cxl_resource {
> + CXL_ACCEL_RES_DPA,
> + CXL_ACCEL_RES_RAM,
> + CXL_ACCEL_RES_PMEM,
> +};
> +
> +struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
> +
> +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
> +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> + enum cxl_resource);
> +#endif
> diff --git a/include/linux/cxl/pci.h b/include/linux/cxl/pci.h
> new file mode 100644
> index 000000000000..c337ae8797e6
> --- /dev/null
> +++ b/include/linux/cxl/pci.h
> @@ -0,0 +1,23 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
Bit bold to claim sole copyright of a cut and paste blob.
Fine to add AMD one, but keep the original copyright as well.
> +
> +#ifndef __CXL_ACCEL_PCI_H
> +#define __CXL_ACCEL_PCI_H
> +
> +/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
> +#define CXL_DVSEC_PCIE_DEVICE 0
> +#define CXL_DVSEC_CAP_OFFSET 0xA
> +#define CXL_DVSEC_MEM_CAPABLE BIT(2)
> +#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
> +#define CXL_DVSEC_CTRL_OFFSET 0xC
> +#define CXL_DVSEC_MEM_ENABLE BIT(2)
> +#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
> +#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
> +#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
> +#define CXL_DVSEC_MEM_ACTIVE BIT(1)
> +#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
> +#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
> +#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
Brackets around (i) to protect against stupid use of the macro.
This is general kernel convention rather than a real problem here.
Sure original code didn't do it but if we are touching the code
might as well fix it ;)
> +#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
> +
> +#endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-13 16:41 ` Jonathan Cameron
@ 2024-09-16 12:03 ` Alejandro Lucero Palau
2024-09-16 12:24 ` Jonathan Cameron
0 siblings, 1 reply; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 12:03 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/13/24 17:41, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:17 +0100
> <alejandro.lucero-palau@amd.com> wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
> Hi Alejandro,
>
> I'm mainly looking at these to get my head back into this support
> for discussions next week but will probably leave
> lots of trivial review feedback as I go.
>
> And to advertise that:
> https://lpc.events/event/18/contributions/1828/
Looking forward to see you there along with other CXL kernel guys.
>> Differientiate Type3, aka memory expanders, from Type2, aka device
> Spell check. Differentiate.
Embarrassing ...I did fix that or I though I did since this was also
pointed out by Dan Williams as well.
I'll definitely fix it for v4.
>> accelerators, with a new function for initializing cxl_dev_state.
>>
>> Create accessors to cxl_dev_state to be used by accel drivers.
>>
>> Add SFC ethernet network driver as the client.
> Minor thing (And others may disagree) but I'd split this to be nice
> to others who might want to backport the type2 support but not
> the sfc changes (as they are supporting some other hardware).
Should I then send incremental sfc changes as well as the API is
introduced or just a final patch with all of it?
>> Based on https://lore.kernel.org/linux-cxl/168592160379.1948938.12863272903570476312.stgit@dwillia2-xfh.jf.intel.com/
> Maybe make that a link tag Link: .... # [1]
> and have
> Based on [1] here.
OK.
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
>
>> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> + enum cxl_resource type)
>> +{
>> + switch (type) {
>> + case CXL_ACCEL_RES_DPA:
>> + cxlds->dpa_res = res;
>> + return 0;
>> + case CXL_ACCEL_RES_RAM:
>> + cxlds->ram_res = res;
>> + return 0;
>> + case CXL_ACCEL_RES_PMEM:
>> + cxlds->pmem_res = res;
>> + return 0;
>> + default:
>> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
> It's an enum, do we need the default? Hence do we need the return value?
>
I think it does not harm and helps with extending the enum without
silently failing if all the places where it is used are not properly
updated.
>> + return -EINVAL;
>> + }
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
>> +
>> static int cxl_memdev_release_file(struct inode *inode, struct file *file)
>> {
>> struct cxl_memdev *cxlmd =
>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>> index 4be35dc22202..742a7b2a1be5 100644
>> --- a/drivers/cxl/pci.c
>> +++ b/drivers/cxl/pci.c
>> @@ -11,6 +11,8 @@
>> #include <linux/pci.h>
>> #include <linux/aer.h>
>> #include <linux/io.h>
>> +#include <linux/cxl/cxl.h>
>> +#include <linux/cxl/pci.h>
>> #include "cxlmem.h"
>> #include "cxlpci.h"
>> #include "cxl.h"
>> @@ -795,6 +797,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> struct cxl_memdev *cxlmd;
>> int i, rc, pmu_count;
>> bool irq_avail;
>> + u16 dvsec;
>>
>> /*
>> * Double check the anonymous union trickery in struct cxl_regs
>> @@ -815,12 +818,14 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> pci_set_drvdata(pdev, cxlds);
>>
>> cxlds->rcd = is_cxl_restricted(pdev);
>> - cxlds->serial = pci_get_dsn(pdev);
>> - cxlds->cxl_dvsec = pci_find_dvsec_capability(
>> - pdev, PCI_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
>> - if (!cxlds->cxl_dvsec)
>> + cxl_set_serial(cxlds, pci_get_dsn(pdev));
>> + dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
>> + CXL_DVSEC_PCIE_DEVICE);
>> + if (!dvsec)
>> dev_warn(&pdev->dev,
>> "Device DVSEC not present, skip CXL.mem init\n");
>> + else
>> + cxl_set_dvsec(cxlds, dvsec);
> Set it unconditionally perhaps. If it's NULL that's fine and then it corresponds
> directly to the previous
OK. I guess keeping the dev_warn. Right?
>>
>> rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
>> if (rc)
>> diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
>> index 6f1a01ded7d4..3a7406aa950c 100644
>> --- a/drivers/net/ethernet/sfc/efx.c
>> +++ b/drivers/net/ethernet/sfc/efx.c
>> @@ -33,6 +33,7 @@
>> #include "selftest.h"
>> #include "sriov.h"
>> #include "efx_devlink.h"
>> +#include "efx_cxl.h"
>>
>> #include "mcdi_port_common.h"
>> #include "mcdi_pcol.h"
>> @@ -899,6 +900,9 @@ static void efx_pci_remove(struct pci_dev *pci_dev)
>> efx_pci_remove_main(efx);
>>
>> efx_fini_io(efx);
>> +
>> + efx_cxl_exit(efx);
>> +
>> pci_dbg(efx->pci_dev, "shutdown successful\n");
>>
>> efx_fini_devlink_and_unlock(efx);
>> @@ -1109,6 +1113,15 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
>> if (rc)
>> goto fail2;
>>
>> + /* A successful cxl initialization implies a CXL region created to be
>> + * used for PIO buffers. If there is no CXL support, or initialization
>> + * fails, efx_cxl_pio_initialised wll be false and legacy PIO buffers
>> + * defined at specific PCI BAR regions will be used.
>> + */
>> + rc = efx_cxl_init(efx);
>> + if (rc)
>> + pci_err(pci_dev, "CXL initialization failed with error %d\n", rc);
> If you are carrying on anyway is pci_info() more appropriate?
> Personally I dislike muddling on in error cases, but understand
> it can be useful on occasion at the cost of more complex flows.
>
>
Not sure. Note this is for the case something went wrong when the device
has CXL support.
It is not fatal, but it is an error.
>> +
>> rc = efx_pci_probe_post_io(efx);
>> if (rc) {
>> /* On failure, retry once immediately.
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
>> new file mode 100644
>> index 000000000000..bba36cbbab22
>> --- /dev/null
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -0,0 +1,86 @@
>> +// SPDX-License-Identifier: GPL-2.0-only
>> +/****************************************************************************
>> + *
>> + * Driver for AMD network controllers and boards
>> + * Copyright (C) 2024, Advanced Micro Devices, Inc.
>> + *
>> + * This program is free software; you can redistribute it and/or modify it
>> + * under the terms of the GNU General Public License version 2 as published
>> + * by the Free Software Foundation, incorporated herein by reference.
>> + */
>> +
>> +#include <linux/cxl/cxl.h>
>> +#include <linux/cxl/pci.h>
>> +#include <linux/pci.h>
>> +
>> +#include "net_driver.h"
>> +#include "efx_cxl.h"
>> +
>> +#define EFX_CTPIO_BUFFER_SIZE (1024 * 1024 * 256)
>> +
>> +int efx_cxl_init(struct efx_nic *efx)
>> +{
>> + struct pci_dev *pci_dev = efx->pci_dev;
>> + struct efx_cxl *cxl;
>> + struct resource res;
>> + u16 dvsec;
>> + int rc;
>> +
>> + efx->efx_cxl_pio_initialised = false;
>> +
>> + dvsec = pci_find_dvsec_capability(pci_dev, PCI_VENDOR_ID_CXL,
>> + CXL_DVSEC_PCIE_DEVICE);
>> +
> Trivial but probably no blank line here. Keeps the error condition tightly
> grouped with the call.
OK
>> + if (!dvsec)
>> + return 0;
>> +
>> + pci_dbg(pci_dev, "CXL_DVSEC_PCIE_DEVICE capability found\n");
>> +
>> + efx->cxl = kzalloc(sizeof(*cxl), GFP_KERNEL);
>> + if (!efx->cxl)
>> + return -ENOMEM;
>> +
>> + cxl = efx->cxl;
> Rather than setting it back to zero in some error paths I'd
> suggest keeping it as local only until you know everything
> succeeded.
>
> cxl = kzalloc(...)
>
It makes sense.
> //maybe also cxlds as then you can use __free() to handle the
> //cleanup paths for both allowing early returns instead
> //of gotos.
Maybe, but using __free is discouraged in network code: 1.6.5 at
https://www.kernel.org/doc/html/latest/process/maintainer-netdev.html
> ...
>
> efx->cxl = cxl;
>
> return 0;
>
>> +
>> + cxl->cxlds = cxl_accel_state_create(&pci_dev->dev);
>> + if (IS_ERR(cxl->cxlds)) {
>> + pci_err(pci_dev, "CXL accel device state failed");
>> + kfree(efx->cxl);
> Use the a separate label below. Error blocks in a given function
> should probably do one or the other between going to labels
> or handling locally. Mixture is harder to read.
OK
>
>> + return -ENOMEM;
>> + }
>> +
>> + cxl_set_dvsec(cxl->cxlds, dvsec);
>> + cxl_set_serial(cxl->cxlds, pci_dev->dev.id);
>> +
>> + res = DEFINE_RES_MEM(0, EFX_CTPIO_BUFFER_SIZE);
>> + if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_DPA)) {
>> + pci_err(pci_dev, "cxl_set_resource DPA failed\n");
>> + rc = -EINVAL;
>> + goto err;
>> + }
>> +
>> + res = DEFINE_RES_MEM_NAMED(0, EFX_CTPIO_BUFFER_SIZE, "ram");
>> + if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_RAM)) {
>> + pci_err(pci_dev, "cxl_set_resource RAM failed\n");
>> + rc = -EINVAL;
>> + goto err;
>> + }
>> +
>> + return 0;
>> +err:
>> + kfree(cxl->cxlds);
>> + kfree(cxl);
>> + efx->cxl = NULL;
>> +
>> + return rc;
>> +}
>> +
>> +void efx_cxl_exit(struct efx_nic *efx)
>> +{
>> + if (efx->cxl) {
>> + kfree(efx->cxl->cxlds);
>> + kfree(efx->cxl);
>> + }
>> +}
>> +
>> +MODULE_IMPORT_NS(CXL);
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.h b/drivers/net/ethernet/sfc/efx_cxl.h
>> new file mode 100644
>> index 000000000000..f57fb2afd124
>> --- /dev/null
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.h
> ...
>
>
>> +struct efx_cxl {
>> + struct cxl_dev_state *cxlds;
>> + struct cxl_memdev *cxlmd;
>> + struct cxl_root_decoder *cxlrd;
>> + struct cxl_port *endpoint;
>> + struct cxl_endpoint_decoder *cxled;
>> + struct cxl_region *efx_region;
> Why is the region efx_ prefixed but nothing else?
> Feels a little random.
>
>> + void __iomem *ctpio_cxl;
>> +};
>> +
>> +int efx_cxl_init(struct efx_nic *efx);
>> +void efx_cxl_exit(struct efx_nic *efx);
>> +#endif
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> new file mode 100644
>> index 000000000000..e78eefa82123
>> --- /dev/null
>> +++ b/include/linux/cxl/cxl.h
>> @@ -0,0 +1,21 @@
>> +/* SPDX-License-Identifier: GPL-2.0 */
>> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
>> +
>> +#ifndef __CXL_H
>> +#define __CXL_H
>> +
>> +#include <linux/device.h>
>> +
>> +enum cxl_resource {
>> + CXL_ACCEL_RES_DPA,
>> + CXL_ACCEL_RES_RAM,
>> + CXL_ACCEL_RES_PMEM,
>> +};
>> +
>> +struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
>> +
>> +void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
>> +void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
>> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> + enum cxl_resource);
>> +#endif
>> diff --git a/include/linux/cxl/pci.h b/include/linux/cxl/pci.h
>> new file mode 100644
>> index 000000000000..c337ae8797e6
>> --- /dev/null
>> +++ b/include/linux/cxl/pci.h
>> @@ -0,0 +1,23 @@
>> +/* SPDX-License-Identifier: GPL-2.0 */
>> +/* Copyright(c) 2024 Advanced Micro Devices, Inc. */
> Bit bold to claim sole copyright of a cut and paste blob.
> Fine to add AMD one, but keep the original copyright as well.
>
Sure.
>> +
>> +#ifndef __CXL_ACCEL_PCI_H
>> +#define __CXL_ACCEL_PCI_H
>> +
>> +/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
>> +#define CXL_DVSEC_PCIE_DEVICE 0
>> +#define CXL_DVSEC_CAP_OFFSET 0xA
>> +#define CXL_DVSEC_MEM_CAPABLE BIT(2)
>> +#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
>> +#define CXL_DVSEC_CTRL_OFFSET 0xC
>> +#define CXL_DVSEC_MEM_ENABLE BIT(2)
>> +#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
>> +#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
>> +#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
>> +#define CXL_DVSEC_MEM_ACTIVE BIT(1)
>> +#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
>> +#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
>> +#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
> Brackets around (i) to protect against stupid use of the macro.
> This is general kernel convention rather than a real problem here.
> Sure original code didn't do it but if we are touching the code
> might as well fix it ;)
I found this warning when checkpatch and I thought it should not be done
then as it was there from a previous patch.
But I agree, I should fix it now.
Thanks!
>
>> +#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
>> +
>> +#endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 01/20] cxl: add type2 device basic support
2024-09-16 12:03 ` Alejandro Lucero Palau
@ 2024-09-16 12:24 ` Jonathan Cameron
0 siblings, 0 replies; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-16 12:24 UTC (permalink / raw)
To: Alejandro Lucero Palau
Cc: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
On Mon, 16 Sep 2024 13:03:10 +0100
Alejandro Lucero Palau <alucerop@amd.com> wrote:
> On 9/13/24 17:41, Jonathan Cameron wrote:
> >> Add SFC ethernet network driver as the client.
> > Minor thing (And others may disagree) but I'd split this to be nice
> > to others who might want to backport the type2 support but not
> > the sfc changes (as they are supporting some other hardware).
>
>
> Should I then send incremental sfc changes as well as the API is
> introduced or just a final patch with all of it?
Given aim is to justify each step for this first user I think
incremental sfc changes do make sense.
>
>
> >> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> >> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> >
> >> +int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> >> + enum cxl_resource type)
> >> +{
> >> + switch (type) {
> >> + case CXL_ACCEL_RES_DPA:
> >> + cxlds->dpa_res = res;
> >> + return 0;
> >> + case CXL_ACCEL_RES_RAM:
> >> + cxlds->ram_res = res;
> >> + return 0;
> >> + case CXL_ACCEL_RES_PMEM:
> >> + cxlds->pmem_res = res;
> >> + return 0;
> >> + default:
> >> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
> > It's an enum, do we need the default? Hence do we need the return value?
> >
>
> I think it does not harm and helps with extending the enum without
> silently failing if all the places where it is used are not properly
> updated.
It won't silently fail. The various build bots love to point out unhandled
cases :) Adding the default means that you'll only see the problem
in runtime testing rather than at build time.
>
>
> >> + return -EINVAL;
> >> + }
> >> +}
> >> +EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
> >> +
> >> static int cxl_memdev_release_file(struct inode *inode, struct file *file)
> >> {
> >> struct cxl_memdev *cxlmd =
> >> + if (!dvsec)
> >> dev_warn(&pdev->dev,
> >> "Device DVSEC not present, skip CXL.mem init\n");
> >> + else
> >> + cxl_set_dvsec(cxlds, dvsec);
> > Set it unconditionally perhaps. If it's NULL that's fine and then it corresponds
> > directly to the previous
>
>
> OK. I guess keeping the dev_warn. Right?
Absolutely.
> >> diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
> >> index 6f1a01ded7d4..3a7406aa950c 100644
> >> --- a/drivers/net/ethernet/sfc/efx.c
> >> +++ b/drivers/net/ethernet/sfc/efx.c
> >> @@ -1109,6 +1113,15 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
> >> if (rc)
> >> goto fail2;
> >>
> >> + /* A successful cxl initialization implies a CXL region created to be
> >> + * used for PIO buffers. If there is no CXL support, or initialization
> >> + * fails, efx_cxl_pio_initialised wll be false and legacy PIO buffers
> >> + * defined at specific PCI BAR regions will be used.
> >> + */
> >> + rc = efx_cxl_init(efx);
> >> + if (rc)
> >> + pci_err(pci_dev, "CXL initialization failed with error %d\n", rc);
> > If you are carrying on anyway is pci_info() more appropriate?
> > Personally I dislike muddling on in error cases, but understand
> > it can be useful on occasion at the cost of more complex flows.
> >
> >
>
> Not sure. Note this is for the case something went wrong when the device
> has CXL support.
>
> It is not fatal, but it is an error.
Fair enough. I don't care that much about this.
>
>
> >> +
> >> rc = efx_pci_probe_post_io(efx);
> >> if (rc) {
> >> /* On failure, retry once immediately.
> >> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> >> new file mode 100644
> >> index 000000000000..bba36cbbab22
> >> --- /dev/null
> >> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> > //maybe also cxlds as then you can use __free() to handle the
> > //cleanup paths for both allowing early returns instead
> > //of gotos.
>
>
> Maybe, but using __free is discouraged in network code: 1.6.5 at
>
> https://www.kernel.org/doc/html/latest/process/maintainer-netdev.html
Fair enough. I've not been keeping up with networking maintainer
preferences recently.
Jonathan
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
2024-09-07 8:18 ` [PATCH v3 01/20] cxl: add type2 device basic support alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-07 18:08 ` kernel test robot
` (2 more replies)
2024-09-07 8:18 ` [PATCH v3 03/20] cxl/pci: add check for validating capabilities alejandro.lucero-palau
` (17 subsequent siblings)
19 siblings, 3 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Type2 devices have some Type3 functionalities as optional like an mbox
or an hdm decoder, and CXL core needs a way to know what an CXL accelerator
implements.
Add a new field for keeping device capabilities as discovered during
initialization.
Add same field to cxl_port which for an endpoint will use those
capabilities discovered previously, and which will be initialized when
calling cxl_port_setup_regs for no endpoints.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/port.c | 9 +++++----
drivers/cxl/core/regs.c | 20 +++++++++++++-------
drivers/cxl/cxl.h | 8 +++++---
drivers/cxl/cxlmem.h | 2 ++
drivers/cxl/pci.c | 9 +++++----
include/linux/cxl/cxl.h | 30 ++++++++++++++++++++++++++++++
6 files changed, 60 insertions(+), 18 deletions(-)
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 1d5007e3795a..39b20ddd0296 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -749,7 +749,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
}
static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
- resource_size_t component_reg_phys)
+ resource_size_t component_reg_phys, u32 *caps)
{
*map = (struct cxl_register_map) {
.host = host,
@@ -763,7 +763,7 @@ static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map
map->reg_type = CXL_REGLOC_RBI_COMPONENT;
map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
- return cxl_setup_regs(map);
+ return cxl_setup_regs(map, caps);
}
static int cxl_port_setup_regs(struct cxl_port *port,
@@ -772,7 +772,7 @@ static int cxl_port_setup_regs(struct cxl_port *port,
if (dev_is_platform(port->uport_dev))
return 0;
return cxl_setup_comp_regs(&port->dev, &port->reg_map,
- component_reg_phys);
+ component_reg_phys, &port->capabilities);
}
static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
@@ -789,7 +789,7 @@ static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
* NULL.
*/
rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map,
- component_reg_phys);
+ component_reg_phys, &dport->port->capabilities);
dport->reg_map.host = host;
return rc;
}
@@ -858,6 +858,7 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host,
port->reg_map = cxlds->reg_map;
port->reg_map.host = &port->dev;
cxlmd->endpoint = port;
+ port->capabilities = cxlds->capabilities;
} else if (parent_dport) {
rc = dev_set_name(dev, "port%d", port->id);
if (rc)
diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
index e1082e749c69..8b8abcadcb93 100644
--- a/drivers/cxl/core/regs.c
+++ b/drivers/cxl/core/regs.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2020 Intel Corporation. */
#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/cxl/cxl.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/pci.h>
@@ -36,7 +37,7 @@
* Probe for component register information and return it in map object.
*/
void cxl_probe_component_regs(struct device *dev, void __iomem *base,
- struct cxl_component_reg_map *map)
+ struct cxl_component_reg_map *map, u32 *caps)
{
int cap, cap_count;
u32 cap_array;
@@ -84,6 +85,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
decoder_cnt = cxl_hdm_decoder_count(hdr);
length = 0x20 * decoder_cnt + 0x10;
rmap = &map->hdm_decoder;
+ *caps |= BIT(CXL_DEV_CAP_HDM);
break;
}
case CXL_CM_CAP_CAP_ID_RAS:
@@ -91,6 +93,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
offset);
length = CXL_RAS_CAPABILITY_LENGTH;
rmap = &map->ras;
+ *caps |= BIT(CXL_DEV_CAP_RAS);
break;
default:
dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id,
@@ -117,7 +120,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_probe_component_regs, CXL);
* Probe for device register information and return it in map object.
*/
void cxl_probe_device_regs(struct device *dev, void __iomem *base,
- struct cxl_device_reg_map *map)
+ struct cxl_device_reg_map *map, u32 *caps)
{
int cap, cap_count;
u64 cap_array;
@@ -146,10 +149,12 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base,
case CXLDEV_CAP_CAP_ID_DEVICE_STATUS:
dev_dbg(dev, "found Status capability (0x%x)\n", offset);
rmap = &map->status;
+ *caps |= BIT(CXL_DEV_CAP_DEV_STATUS);
break;
case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX:
dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset);
rmap = &map->mbox;
+ *caps |= BIT(CXL_DEV_CAP_MAILBOX_PRIMARY);
break;
case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX:
dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset);
@@ -157,6 +162,7 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base,
case CXLDEV_CAP_CAP_ID_MEMDEV:
dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset);
rmap = &map->memdev;
+ *caps |= BIT(CXL_DEV_CAP_MEMDEV);
break;
default:
if (cap_id >= 0x8000)
@@ -421,7 +427,7 @@ static void cxl_unmap_regblock(struct cxl_register_map *map)
map->base = NULL;
}
-static int cxl_probe_regs(struct cxl_register_map *map)
+static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
{
struct cxl_component_reg_map *comp_map;
struct cxl_device_reg_map *dev_map;
@@ -431,12 +437,12 @@ static int cxl_probe_regs(struct cxl_register_map *map)
switch (map->reg_type) {
case CXL_REGLOC_RBI_COMPONENT:
comp_map = &map->component_map;
- cxl_probe_component_regs(host, base, comp_map);
+ cxl_probe_component_regs(host, base, comp_map, caps);
dev_dbg(host, "Set up component registers\n");
break;
case CXL_REGLOC_RBI_MEMDEV:
dev_map = &map->device_map;
- cxl_probe_device_regs(host, base, dev_map);
+ cxl_probe_device_regs(host, base, dev_map, caps);
if (!dev_map->status.valid || !dev_map->mbox.valid ||
!dev_map->memdev.valid) {
dev_err(host, "registers not found: %s%s%s\n",
@@ -455,7 +461,7 @@ static int cxl_probe_regs(struct cxl_register_map *map)
return 0;
}
-int cxl_setup_regs(struct cxl_register_map *map)
+int cxl_setup_regs(struct cxl_register_map *map, u32 *caps)
{
int rc;
@@ -463,7 +469,7 @@ int cxl_setup_regs(struct cxl_register_map *map)
if (rc)
return rc;
- rc = cxl_probe_regs(map);
+ rc = cxl_probe_regs(map, caps);
cxl_unmap_regblock(map);
return rc;
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 9afb407d438f..07c153aa3d77 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -284,9 +284,9 @@ struct cxl_register_map {
};
void cxl_probe_component_regs(struct device *dev, void __iomem *base,
- struct cxl_component_reg_map *map);
+ struct cxl_component_reg_map *map, u32 *caps);
void cxl_probe_device_regs(struct device *dev, void __iomem *base,
- struct cxl_device_reg_map *map);
+ struct cxl_device_reg_map *map, u32 *caps);
int cxl_map_component_regs(const struct cxl_register_map *map,
struct cxl_component_regs *regs,
unsigned long map_mask);
@@ -300,7 +300,7 @@ int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type,
struct cxl_register_map *map, int index);
int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
struct cxl_register_map *map);
-int cxl_setup_regs(struct cxl_register_map *map);
+int cxl_setup_regs(struct cxl_register_map *map, u32 *caps);
struct cxl_dport;
resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
struct cxl_dport *dport);
@@ -600,6 +600,7 @@ struct cxl_dax_region {
* @cdat: Cached CDAT data
* @cdat_available: Should a CDAT attribute be available in sysfs
* @pci_latency: Upstream latency in picoseconds
+ * @capabilities: those capabilities as defined in device mapped registers
*/
struct cxl_port {
struct device dev;
@@ -623,6 +624,7 @@ struct cxl_port {
} cdat;
bool cdat_available;
long pci_latency;
+ u32 capabilities;
};
/**
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index afb53d058d62..37c043100300 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -424,6 +424,7 @@ struct cxl_dpa_perf {
* @ram_res: Active Volatile memory capacity configuration
* @serial: PCIe Device Serial Number
* @type: Generic Memory Class device or Vendor Specific Memory device
+ * @capabilities: those capabilities as defined in device mapped registers
*/
struct cxl_dev_state {
struct device *dev;
@@ -438,6 +439,7 @@ struct cxl_dev_state {
struct resource ram_res;
u64 serial;
enum cxl_devtype type;
+ u32 capabilities;
};
/**
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 742a7b2a1be5..58f325019886 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -503,7 +503,7 @@ static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
}
static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
- struct cxl_register_map *map)
+ struct cxl_register_map *map, u32 *caps)
{
int rc;
@@ -520,7 +520,7 @@ static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
if (rc)
return rc;
- return cxl_setup_regs(map);
+ return cxl_setup_regs(map, caps);
}
static int cxl_pci_ras_unmask(struct pci_dev *pdev)
@@ -827,7 +827,8 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
else
cxl_set_dvsec(cxlds, dvsec);
- rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
+ rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map,
+ &cxlds->capabilities);
if (rc)
return rc;
@@ -840,7 +841,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
* still be useful for management functions so don't return an error.
*/
rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
- &cxlds->reg_map);
+ &cxlds->reg_map, &cxlds->capabilities);
if (rc)
dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
else if (!cxlds->reg_map.component_map.ras.valid)
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index e78eefa82123..930b1b9c1d6a 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -12,6 +12,36 @@ enum cxl_resource {
CXL_ACCEL_RES_PMEM,
};
+/* Capabilities as defined for:
+ *
+ * Component Registers (Table 8-22 CXL 3.0 specification)
+ * Device Registers (8.2.8.2.1 CXL 3.0 specification)
+ */
+
+enum cxl_dev_cap {
+ /* capabilities from Component Registers */
+ CXL_DEV_CAP_RAS,
+ CXL_DEV_CAP_SEC,
+ CXL_DEV_CAP_LINK,
+ CXL_DEV_CAP_HDM,
+ CXL_DEV_CAP_SEC_EXT,
+ CXL_DEV_CAP_IDE,
+ CXL_DEV_CAP_SNOOP_FILTER,
+ CXL_DEV_CAP_TIMEOUT_AND_ISOLATION,
+ CXL_DEV_CAP_CACHEMEM_EXT,
+ CXL_DEV_CAP_BI_ROUTE_TABLE,
+ CXL_DEV_CAP_BI_DECODER,
+ CXL_DEV_CAP_CACHEID_ROUTE_TABLE,
+ CXL_DEV_CAP_CACHEID_DECODER,
+ CXL_DEV_CAP_HDM_EXT,
+ CXL_DEV_CAP_METADATA_EXT,
+ /* capabilities from Device Registers */
+ CXL_DEV_CAP_DEV_STATUS,
+ CXL_DEV_CAP_MAILBOX_PRIMARY,
+ CXL_DEV_CAP_MAILBOX_SECONDARY,
+ CXL_DEV_CAP_MEMDEV,
+};
+
struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port
2024-09-07 8:18 ` [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port alejandro.lucero-palau
@ 2024-09-07 18:08 ` kernel test robot
2024-09-11 22:17 ` Dave Jiang
2024-09-13 17:25 ` Jonathan Cameron
2 siblings, 0 replies; 88+ messages in thread
From: kernel test robot @ 2024-09-07 18:08 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: oe-kbuild-all, Alejandro Lucero
Hi,
kernel test robot noticed the following build warnings:
[auto build test WARNING on cxl/next]
[also build test WARNING on linus/master v6.11-rc6 next-20240906]
[cannot apply to cxl/pending horms-ipvs/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/alejandro-lucero-palau-amd-com/cxl-add-type2-device-basic-support/20240907-162231
base: https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git next
patch link: https://lore.kernel.org/r/20240907081836.5801-3-alejandro.lucero-palau%40amd.com
patch subject: [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port
config: xtensa-randconfig-r073-20240908 (https://download.01.org/0day-ci/archive/20240908/202409080140.BHrsmdob-lkp@intel.com/config)
compiler: xtensa-linux-gcc (GCC) 14.1.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240908/202409080140.BHrsmdob-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202409080140.BHrsmdob-lkp@intel.com/
All warnings (new ones prefixed by >>):
>> drivers/cxl/core/regs.c:41: warning: Function parameter or struct member 'caps' not described in 'cxl_probe_component_regs'
>> drivers/cxl/core/regs.c:124: warning: Function parameter or struct member 'caps' not described in 'cxl_probe_device_regs'
vim +41 drivers/cxl/core/regs.c
fa89248e669d58 Robert Richter 2022-10-18 13
2b922a9d064f8e Dan Williams 2021-09-03 14 /**
2b922a9d064f8e Dan Williams 2021-09-03 15 * DOC: cxl registers
2b922a9d064f8e Dan Williams 2021-09-03 16 *
2b922a9d064f8e Dan Williams 2021-09-03 17 * CXL device capabilities are enumerated by PCI DVSEC (Designated
2b922a9d064f8e Dan Williams 2021-09-03 18 * Vendor-specific) and / or descriptors provided by platform firmware.
2b922a9d064f8e Dan Williams 2021-09-03 19 * They can be defined as a set like the device and component registers
2b922a9d064f8e Dan Williams 2021-09-03 20 * mandated by CXL Section 8.1.12.2 Memory Device PCIe Capabilities and
2b922a9d064f8e Dan Williams 2021-09-03 21 * Extended Capabilities, or they can be individual capabilities
2b922a9d064f8e Dan Williams 2021-09-03 22 * appended to bridged and endpoint devices.
2b922a9d064f8e Dan Williams 2021-09-03 23 *
2b922a9d064f8e Dan Williams 2021-09-03 24 * Provide common infrastructure for enumerating and mapping these
2b922a9d064f8e Dan Williams 2021-09-03 25 * discrete capabilities.
2b922a9d064f8e Dan Williams 2021-09-03 26 */
2b922a9d064f8e Dan Williams 2021-09-03 27
0f06157e0135f5 Dan Williams 2021-08-03 28 /**
0f06157e0135f5 Dan Williams 2021-08-03 29 * cxl_probe_component_regs() - Detect CXL Component register blocks
0f06157e0135f5 Dan Williams 2021-08-03 30 * @dev: Host device of the @base mapping
0f06157e0135f5 Dan Williams 2021-08-03 31 * @base: Mapping containing the HDM Decoder Capability Header
0f06157e0135f5 Dan Williams 2021-08-03 32 * @map: Map object describing the register block information found
0f06157e0135f5 Dan Williams 2021-08-03 33 *
0f06157e0135f5 Dan Williams 2021-08-03 34 * See CXL 2.0 8.2.4 Component Register Layout and Definition
0f06157e0135f5 Dan Williams 2021-08-03 35 * See CXL 2.0 8.2.5.5 CXL Device Register Interface
0f06157e0135f5 Dan Williams 2021-08-03 36 *
0f06157e0135f5 Dan Williams 2021-08-03 37 * Probe for component register information and return it in map object.
0f06157e0135f5 Dan Williams 2021-08-03 38 */
0f06157e0135f5 Dan Williams 2021-08-03 39 void cxl_probe_component_regs(struct device *dev, void __iomem *base,
98279f48d53f4f Alejandro Lucero 2024-09-07 40 struct cxl_component_reg_map *map, u32 *caps)
0f06157e0135f5 Dan Williams 2021-08-03 @41 {
0f06157e0135f5 Dan Williams 2021-08-03 42 int cap, cap_count;
74b0fe80409733 Jonathan Cameron 2022-02-01 43 u32 cap_array;
0f06157e0135f5 Dan Williams 2021-08-03 44
0f06157e0135f5 Dan Williams 2021-08-03 45 *map = (struct cxl_component_reg_map) { 0 };
0f06157e0135f5 Dan Williams 2021-08-03 46
0f06157e0135f5 Dan Williams 2021-08-03 47 /*
0f06157e0135f5 Dan Williams 2021-08-03 48 * CXL.cache and CXL.mem registers are at offset 0x1000 as defined in
0f06157e0135f5 Dan Williams 2021-08-03 49 * CXL 2.0 8.2.4 Table 141.
0f06157e0135f5 Dan Williams 2021-08-03 50 */
0f06157e0135f5 Dan Williams 2021-08-03 51 base += CXL_CM_OFFSET;
0f06157e0135f5 Dan Williams 2021-08-03 52
74b0fe80409733 Jonathan Cameron 2022-02-01 53 cap_array = readl(base + CXL_CM_CAP_HDR_OFFSET);
0f06157e0135f5 Dan Williams 2021-08-03 54
0f06157e0135f5 Dan Williams 2021-08-03 55 if (FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, cap_array) != CM_CAP_HDR_CAP_ID) {
0f06157e0135f5 Dan Williams 2021-08-03 56 dev_err(dev,
d621bc2e7282f9 Dan Williams 2022-01-23 57 "Couldn't locate the CXL.cache and CXL.mem capability array header.\n");
0f06157e0135f5 Dan Williams 2021-08-03 58 return;
0f06157e0135f5 Dan Williams 2021-08-03 59 }
0f06157e0135f5 Dan Williams 2021-08-03 60
0f06157e0135f5 Dan Williams 2021-08-03 61 /* It's assumed that future versions will be backward compatible */
0f06157e0135f5 Dan Williams 2021-08-03 62 cap_count = FIELD_GET(CXL_CM_CAP_HDR_ARRAY_SIZE_MASK, cap_array);
0f06157e0135f5 Dan Williams 2021-08-03 63
0f06157e0135f5 Dan Williams 2021-08-03 64 for (cap = 1; cap <= cap_count; cap++) {
0f06157e0135f5 Dan Williams 2021-08-03 65 void __iomem *register_block;
af2dfef854aa6a Dan Williams 2022-11-29 66 struct cxl_reg_map *rmap;
0f06157e0135f5 Dan Williams 2021-08-03 67 u16 cap_id, offset;
af2dfef854aa6a Dan Williams 2022-11-29 68 u32 length, hdr;
0f06157e0135f5 Dan Williams 2021-08-03 69
0f06157e0135f5 Dan Williams 2021-08-03 70 hdr = readl(base + cap * 0x4);
0f06157e0135f5 Dan Williams 2021-08-03 71
0f06157e0135f5 Dan Williams 2021-08-03 72 cap_id = FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, hdr);
0f06157e0135f5 Dan Williams 2021-08-03 73 offset = FIELD_GET(CXL_CM_CAP_PTR_MASK, hdr);
0f06157e0135f5 Dan Williams 2021-08-03 74 register_block = base + offset;
af2dfef854aa6a Dan Williams 2022-11-29 75 hdr = readl(register_block);
0f06157e0135f5 Dan Williams 2021-08-03 76
af2dfef854aa6a Dan Williams 2022-11-29 77 rmap = NULL;
0f06157e0135f5 Dan Williams 2021-08-03 78 switch (cap_id) {
af2dfef854aa6a Dan Williams 2022-11-29 79 case CXL_CM_CAP_CAP_ID_HDM: {
af2dfef854aa6a Dan Williams 2022-11-29 80 int decoder_cnt;
af2dfef854aa6a Dan Williams 2022-11-29 81
0f06157e0135f5 Dan Williams 2021-08-03 82 dev_dbg(dev, "found HDM decoder capability (0x%x)\n",
0f06157e0135f5 Dan Williams 2021-08-03 83 offset);
0f06157e0135f5 Dan Williams 2021-08-03 84
0f06157e0135f5 Dan Williams 2021-08-03 85 decoder_cnt = cxl_hdm_decoder_count(hdr);
0f06157e0135f5 Dan Williams 2021-08-03 86 length = 0x20 * decoder_cnt + 0x10;
af2dfef854aa6a Dan Williams 2022-11-29 87 rmap = &map->hdm_decoder;
98279f48d53f4f Alejandro Lucero 2024-09-07 88 *caps |= BIT(CXL_DEV_CAP_HDM);
0f06157e0135f5 Dan Williams 2021-08-03 89 break;
af2dfef854aa6a Dan Williams 2022-11-29 90 }
bd09626b39dff9 Dan Williams 2022-11-29 91 case CXL_CM_CAP_CAP_ID_RAS:
bd09626b39dff9 Dan Williams 2022-11-29 92 dev_dbg(dev, "found RAS capability (0x%x)\n",
bd09626b39dff9 Dan Williams 2022-11-29 93 offset);
bd09626b39dff9 Dan Williams 2022-11-29 94 length = CXL_RAS_CAPABILITY_LENGTH;
bd09626b39dff9 Dan Williams 2022-11-29 95 rmap = &map->ras;
98279f48d53f4f Alejandro Lucero 2024-09-07 96 *caps |= BIT(CXL_DEV_CAP_RAS);
0f06157e0135f5 Dan Williams 2021-08-03 97 break;
0f06157e0135f5 Dan Williams 2021-08-03 98 default:
0f06157e0135f5 Dan Williams 2021-08-03 99 dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id,
0f06157e0135f5 Dan Williams 2021-08-03 100 offset);
0f06157e0135f5 Dan Williams 2021-08-03 101 break;
0f06157e0135f5 Dan Williams 2021-08-03 102 }
af2dfef854aa6a Dan Williams 2022-11-29 103
af2dfef854aa6a Dan Williams 2022-11-29 104 if (!rmap)
af2dfef854aa6a Dan Williams 2022-11-29 105 continue;
af2dfef854aa6a Dan Williams 2022-11-29 106 rmap->valid = true;
a1554e9cac5ea0 Dan Williams 2022-11-29 107 rmap->id = cap_id;
af2dfef854aa6a Dan Williams 2022-11-29 108 rmap->offset = CXL_CM_OFFSET + offset;
af2dfef854aa6a Dan Williams 2022-11-29 109 rmap->size = length;
0f06157e0135f5 Dan Williams 2021-08-03 110 }
0f06157e0135f5 Dan Williams 2021-08-03 111 }
affec782742e08 Dan Williams 2021-11-12 112 EXPORT_SYMBOL_NS_GPL(cxl_probe_component_regs, CXL);
0f06157e0135f5 Dan Williams 2021-08-03 113
0f06157e0135f5 Dan Williams 2021-08-03 114 /**
0f06157e0135f5 Dan Williams 2021-08-03 115 * cxl_probe_device_regs() - Detect CXL Device register blocks
0f06157e0135f5 Dan Williams 2021-08-03 116 * @dev: Host device of the @base mapping
0f06157e0135f5 Dan Williams 2021-08-03 117 * @base: Mapping of CXL 2.0 8.2.8 CXL Device Register Interface
0f06157e0135f5 Dan Williams 2021-08-03 118 * @map: Map object describing the register block information found
0f06157e0135f5 Dan Williams 2021-08-03 119 *
0f06157e0135f5 Dan Williams 2021-08-03 120 * Probe for device register information and return it in map object.
0f06157e0135f5 Dan Williams 2021-08-03 121 */
0f06157e0135f5 Dan Williams 2021-08-03 122 void cxl_probe_device_regs(struct device *dev, void __iomem *base,
98279f48d53f4f Alejandro Lucero 2024-09-07 123 struct cxl_device_reg_map *map, u32 *caps)
0f06157e0135f5 Dan Williams 2021-08-03 @124 {
0f06157e0135f5 Dan Williams 2021-08-03 125 int cap, cap_count;
0f06157e0135f5 Dan Williams 2021-08-03 126 u64 cap_array;
0f06157e0135f5 Dan Williams 2021-08-03 127
0f06157e0135f5 Dan Williams 2021-08-03 128 *map = (struct cxl_device_reg_map){ 0 };
0f06157e0135f5 Dan Williams 2021-08-03 129
0f06157e0135f5 Dan Williams 2021-08-03 130 cap_array = readq(base + CXLDEV_CAP_ARRAY_OFFSET);
0f06157e0135f5 Dan Williams 2021-08-03 131 if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) !=
0f06157e0135f5 Dan Williams 2021-08-03 132 CXLDEV_CAP_ARRAY_CAP_ID)
0f06157e0135f5 Dan Williams 2021-08-03 133 return;
0f06157e0135f5 Dan Williams 2021-08-03 134
0f06157e0135f5 Dan Williams 2021-08-03 135 cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array);
0f06157e0135f5 Dan Williams 2021-08-03 136
0f06157e0135f5 Dan Williams 2021-08-03 137 for (cap = 1; cap <= cap_count; cap++) {
af2dfef854aa6a Dan Williams 2022-11-29 138 struct cxl_reg_map *rmap;
0f06157e0135f5 Dan Williams 2021-08-03 139 u32 offset, length;
0f06157e0135f5 Dan Williams 2021-08-03 140 u16 cap_id;
0f06157e0135f5 Dan Williams 2021-08-03 141
0f06157e0135f5 Dan Williams 2021-08-03 142 cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK,
0f06157e0135f5 Dan Williams 2021-08-03 143 readl(base + cap * 0x10));
0f06157e0135f5 Dan Williams 2021-08-03 144 offset = readl(base + cap * 0x10 + 0x4);
0f06157e0135f5 Dan Williams 2021-08-03 145 length = readl(base + cap * 0x10 + 0x8);
0f06157e0135f5 Dan Williams 2021-08-03 146
af2dfef854aa6a Dan Williams 2022-11-29 147 rmap = NULL;
0f06157e0135f5 Dan Williams 2021-08-03 148 switch (cap_id) {
0f06157e0135f5 Dan Williams 2021-08-03 149 case CXLDEV_CAP_CAP_ID_DEVICE_STATUS:
0f06157e0135f5 Dan Williams 2021-08-03 150 dev_dbg(dev, "found Status capability (0x%x)\n", offset);
af2dfef854aa6a Dan Williams 2022-11-29 151 rmap = &map->status;
98279f48d53f4f Alejandro Lucero 2024-09-07 152 *caps |= BIT(CXL_DEV_CAP_DEV_STATUS);
0f06157e0135f5 Dan Williams 2021-08-03 153 break;
0f06157e0135f5 Dan Williams 2021-08-03 154 case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX:
0f06157e0135f5 Dan Williams 2021-08-03 155 dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset);
af2dfef854aa6a Dan Williams 2022-11-29 156 rmap = &map->mbox;
98279f48d53f4f Alejandro Lucero 2024-09-07 157 *caps |= BIT(CXL_DEV_CAP_MAILBOX_PRIMARY);
0f06157e0135f5 Dan Williams 2021-08-03 158 break;
0f06157e0135f5 Dan Williams 2021-08-03 159 case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX:
0f06157e0135f5 Dan Williams 2021-08-03 160 dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset);
0f06157e0135f5 Dan Williams 2021-08-03 161 break;
0f06157e0135f5 Dan Williams 2021-08-03 162 case CXLDEV_CAP_CAP_ID_MEMDEV:
0f06157e0135f5 Dan Williams 2021-08-03 163 dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset);
af2dfef854aa6a Dan Williams 2022-11-29 164 rmap = &map->memdev;
98279f48d53f4f Alejandro Lucero 2024-09-07 165 *caps |= BIT(CXL_DEV_CAP_MEMDEV);
0f06157e0135f5 Dan Williams 2021-08-03 166 break;
0f06157e0135f5 Dan Williams 2021-08-03 167 default:
0f06157e0135f5 Dan Williams 2021-08-03 168 if (cap_id >= 0x8000)
0f06157e0135f5 Dan Williams 2021-08-03 169 dev_dbg(dev, "Vendor cap ID: %#x offset: %#x\n", cap_id, offset);
0f06157e0135f5 Dan Williams 2021-08-03 170 else
0f06157e0135f5 Dan Williams 2021-08-03 171 dev_dbg(dev, "Unknown cap ID: %#x offset: %#x\n", cap_id, offset);
0f06157e0135f5 Dan Williams 2021-08-03 172 break;
0f06157e0135f5 Dan Williams 2021-08-03 173 }
af2dfef854aa6a Dan Williams 2022-11-29 174
af2dfef854aa6a Dan Williams 2022-11-29 175 if (!rmap)
af2dfef854aa6a Dan Williams 2022-11-29 176 continue;
af2dfef854aa6a Dan Williams 2022-11-29 177 rmap->valid = true;
a1554e9cac5ea0 Dan Williams 2022-11-29 178 rmap->id = cap_id;
af2dfef854aa6a Dan Williams 2022-11-29 179 rmap->offset = offset;
af2dfef854aa6a Dan Williams 2022-11-29 180 rmap->size = length;
0f06157e0135f5 Dan Williams 2021-08-03 181 }
0f06157e0135f5 Dan Williams 2021-08-03 182 }
affec782742e08 Dan Williams 2021-11-12 183 EXPORT_SYMBOL_NS_GPL(cxl_probe_device_regs, CXL);
0f06157e0135f5 Dan Williams 2021-08-03 184
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port
2024-09-07 8:18 ` [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port alejandro.lucero-palau
2024-09-07 18:08 ` kernel test robot
@ 2024-09-11 22:17 ` Dave Jiang
2024-09-16 8:36 ` Alejandro Lucero Palau
2024-09-13 17:25 ` Jonathan Cameron
2 siblings, 1 reply; 88+ messages in thread
From: Dave Jiang @ 2024-09-11 22:17 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Type2 devices have some Type3 functionalities as optional like an mbox
> or an hdm decoder, and CXL core needs a way to know what an CXL accelerator
> implements.
>
> Add a new field for keeping device capabilities as discovered during
> initialization.
>
> Add same field to cxl_port which for an endpoint will use those
> capabilities discovered previously, and which will be initialized when
> calling cxl_port_setup_regs for no endpoints.
I don't quite understand what you are trying to say here.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/port.c | 9 +++++----
> drivers/cxl/core/regs.c | 20 +++++++++++++-------
> drivers/cxl/cxl.h | 8 +++++---
> drivers/cxl/cxlmem.h | 2 ++
> drivers/cxl/pci.c | 9 +++++----
> include/linux/cxl/cxl.h | 30 ++++++++++++++++++++++++++++++
> 6 files changed, 60 insertions(+), 18 deletions(-)
>
> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
> index 1d5007e3795a..39b20ddd0296 100644
> --- a/drivers/cxl/core/port.c
> +++ b/drivers/cxl/core/port.c
> @@ -749,7 +749,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
> }
>
> static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
> - resource_size_t component_reg_phys)
> + resource_size_t component_reg_phys, u32 *caps)
> {
> *map = (struct cxl_register_map) {
> .host = host,
> @@ -763,7 +763,7 @@ static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map
> map->reg_type = CXL_REGLOC_RBI_COMPONENT;
> map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
>
> - return cxl_setup_regs(map);
> + return cxl_setup_regs(map, caps);
> }
>
> static int cxl_port_setup_regs(struct cxl_port *port,
> @@ -772,7 +772,7 @@ static int cxl_port_setup_regs(struct cxl_port *port,
> if (dev_is_platform(port->uport_dev))
> return 0;
> return cxl_setup_comp_regs(&port->dev, &port->reg_map,
> - component_reg_phys);
> + component_reg_phys, &port->capabilities);
> }
>
> static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
> @@ -789,7 +789,7 @@ static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
> * NULL.
> */
> rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map,
> - component_reg_phys);
> + component_reg_phys, &dport->port->capabilities);
> dport->reg_map.host = host;
> return rc;
> }
> @@ -858,6 +858,7 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host,
> port->reg_map = cxlds->reg_map;
> port->reg_map.host = &port->dev;
> cxlmd->endpoint = port;
> + port->capabilities = cxlds->capabilities;
> } else if (parent_dport) {
> rc = dev_set_name(dev, "port%d", port->id);
> if (rc)
> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
> index e1082e749c69..8b8abcadcb93 100644
> --- a/drivers/cxl/core/regs.c
> +++ b/drivers/cxl/core/regs.c
> @@ -1,6 +1,7 @@
> // SPDX-License-Identifier: GPL-2.0-only
> /* Copyright(c) 2020 Intel Corporation. */
> #include <linux/io-64-nonatomic-lo-hi.h>
> +#include <linux/cxl/cxl.h>
> #include <linux/device.h>
> #include <linux/slab.h>
> #include <linux/pci.h>
> @@ -36,7 +37,7 @@
> * Probe for component register information and return it in map object.
> */
> void cxl_probe_component_regs(struct device *dev, void __iomem *base,
> - struct cxl_component_reg_map *map)
> + struct cxl_component_reg_map *map, u32 *caps)
> {
> int cap, cap_count;
> u32 cap_array;
> @@ -84,6 +85,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
> decoder_cnt = cxl_hdm_decoder_count(hdr);
> length = 0x20 * decoder_cnt + 0x10;
> rmap = &map->hdm_decoder;
> + *caps |= BIT(CXL_DEV_CAP_HDM);
> break;
> }
> case CXL_CM_CAP_CAP_ID_RAS:
> @@ -91,6 +93,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
> offset);
> length = CXL_RAS_CAPABILITY_LENGTH;
> rmap = &map->ras;
> + *caps |= BIT(CXL_DEV_CAP_RAS);
> break;
> default:
> dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id,
> @@ -117,7 +120,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_probe_component_regs, CXL);
> * Probe for device register information and return it in map object.
> */
> void cxl_probe_device_regs(struct device *dev, void __iomem *base,
> - struct cxl_device_reg_map *map)
> + struct cxl_device_reg_map *map, u32 *caps)
> {
> int cap, cap_count;
> u64 cap_array;
> @@ -146,10 +149,12 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base,
> case CXLDEV_CAP_CAP_ID_DEVICE_STATUS:
> dev_dbg(dev, "found Status capability (0x%x)\n", offset);
> rmap = &map->status;
> + *caps |= BIT(CXL_DEV_CAP_DEV_STATUS);
> break;
> case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX:
> dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset);
> rmap = &map->mbox;
> + *caps |= BIT(CXL_DEV_CAP_MAILBOX_PRIMARY);
> break;
> case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX:
> dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset);
> @@ -157,6 +162,7 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base,
> case CXLDEV_CAP_CAP_ID_MEMDEV:
> dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset);
> rmap = &map->memdev;
> + *caps |= BIT(CXL_DEV_CAP_MEMDEV);
> break;
> default:
> if (cap_id >= 0x8000)
> @@ -421,7 +427,7 @@ static void cxl_unmap_regblock(struct cxl_register_map *map)
> map->base = NULL;
> }
>
> -static int cxl_probe_regs(struct cxl_register_map *map)
> +static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
> {
> struct cxl_component_reg_map *comp_map;
> struct cxl_device_reg_map *dev_map;
> @@ -431,12 +437,12 @@ static int cxl_probe_regs(struct cxl_register_map *map)
> switch (map->reg_type) {
> case CXL_REGLOC_RBI_COMPONENT:
> comp_map = &map->component_map;
> - cxl_probe_component_regs(host, base, comp_map);
> + cxl_probe_component_regs(host, base, comp_map, caps);
> dev_dbg(host, "Set up component registers\n");
> break;
> case CXL_REGLOC_RBI_MEMDEV:
> dev_map = &map->device_map;
> - cxl_probe_device_regs(host, base, dev_map);
> + cxl_probe_device_regs(host, base, dev_map, caps);
> if (!dev_map->status.valid || !dev_map->mbox.valid ||
> !dev_map->memdev.valid) {
> dev_err(host, "registers not found: %s%s%s\n",
> @@ -455,7 +461,7 @@ static int cxl_probe_regs(struct cxl_register_map *map)
> return 0;
> }
>
> -int cxl_setup_regs(struct cxl_register_map *map)
> +int cxl_setup_regs(struct cxl_register_map *map, u32 *caps)
> {
> int rc;
>
> @@ -463,7 +469,7 @@ int cxl_setup_regs(struct cxl_register_map *map)
> if (rc)
> return rc;
>
> - rc = cxl_probe_regs(map);
> + rc = cxl_probe_regs(map, caps);
> cxl_unmap_regblock(map);
>
> return rc;
> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
> index 9afb407d438f..07c153aa3d77 100644
> --- a/drivers/cxl/cxl.h
> +++ b/drivers/cxl/cxl.h
> @@ -284,9 +284,9 @@ struct cxl_register_map {
> };
>
> void cxl_probe_component_regs(struct device *dev, void __iomem *base,
> - struct cxl_component_reg_map *map);
> + struct cxl_component_reg_map *map, u32 *caps);
> void cxl_probe_device_regs(struct device *dev, void __iomem *base,
> - struct cxl_device_reg_map *map);
> + struct cxl_device_reg_map *map, u32 *caps);
> int cxl_map_component_regs(const struct cxl_register_map *map,
> struct cxl_component_regs *regs,
> unsigned long map_mask);
> @@ -300,7 +300,7 @@ int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type,
> struct cxl_register_map *map, int index);
> int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
> struct cxl_register_map *map);
> -int cxl_setup_regs(struct cxl_register_map *map);
> +int cxl_setup_regs(struct cxl_register_map *map, u32 *caps);
> struct cxl_dport;
> resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
> struct cxl_dport *dport);
> @@ -600,6 +600,7 @@ struct cxl_dax_region {
> * @cdat: Cached CDAT data
> * @cdat_available: Should a CDAT attribute be available in sysfs
> * @pci_latency: Upstream latency in picoseconds
> + * @capabilities: those capabilities as defined in device mapped registers
> */
> struct cxl_port {
> struct device dev;
> @@ -623,6 +624,7 @@ struct cxl_port {
> } cdat;
> bool cdat_available;
> long pci_latency;
> + u32 capabilities;
> };
>
> /**
> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> index afb53d058d62..37c043100300 100644
> --- a/drivers/cxl/cxlmem.h
> +++ b/drivers/cxl/cxlmem.h
> @@ -424,6 +424,7 @@ struct cxl_dpa_perf {
> * @ram_res: Active Volatile memory capacity configuration
> * @serial: PCIe Device Serial Number
> * @type: Generic Memory Class device or Vendor Specific Memory device
> + * @capabilities: those capabilities as defined in device mapped registers
> */
> struct cxl_dev_state {
> struct device *dev;
> @@ -438,6 +439,7 @@ struct cxl_dev_state {
> struct resource ram_res;
> u64 serial;
> enum cxl_devtype type;
> + u32 capabilities;
> };
>
> /**
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 742a7b2a1be5..58f325019886 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -503,7 +503,7 @@ static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
> }
>
> static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
> - struct cxl_register_map *map)
> + struct cxl_register_map *map, u32 *caps)
> {
> int rc;
>
> @@ -520,7 +520,7 @@ static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
> if (rc)
> return rc;
>
> - return cxl_setup_regs(map);
> + return cxl_setup_regs(map, caps);
> }
>
> static int cxl_pci_ras_unmask(struct pci_dev *pdev)
> @@ -827,7 +827,8 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> else
> cxl_set_dvsec(cxlds, dvsec);
>
> - rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map,
> + &cxlds->capabilities);
> if (rc)
> return rc;
>
> @@ -840,7 +841,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> * still be useful for management functions so don't return an error.
> */
> rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
> - &cxlds->reg_map);
> + &cxlds->reg_map, &cxlds->capabilities);
> if (rc)
> dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
> else if (!cxlds->reg_map.component_map.ras.valid)
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index e78eefa82123..930b1b9c1d6a 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -12,6 +12,36 @@ enum cxl_resource {
> CXL_ACCEL_RES_PMEM,
> };
>
> +/* Capabilities as defined for:
> + *
> + * Component Registers (Table 8-22 CXL 3.0 specification)
> + * Device Registers (8.2.8.2.1 CXL 3.0 specification)
Should just use 3.1 since that's the latest spec.
> + */
> +
> +enum cxl_dev_cap {
> + /* capabilities from Component Registers */
> + CXL_DEV_CAP_RAS,
> + CXL_DEV_CAP_SEC,
> + CXL_DEV_CAP_LINK,
> + CXL_DEV_CAP_HDM,
> + CXL_DEV_CAP_SEC_EXT,
> + CXL_DEV_CAP_IDE,
> + CXL_DEV_CAP_SNOOP_FILTER,
> + CXL_DEV_CAP_TIMEOUT_AND_ISOLATION,
> + CXL_DEV_CAP_CACHEMEM_EXT,
> + CXL_DEV_CAP_BI_ROUTE_TABLE,
> + CXL_DEV_CAP_BI_DECODER,
> + CXL_DEV_CAP_CACHEID_ROUTE_TABLE,
> + CXL_DEV_CAP_CACHEID_DECODER,
> + CXL_DEV_CAP_HDM_EXT,
> + CXL_DEV_CAP_METADATA_EXT,
> + /* capabilities from Device Registers */
> + CXL_DEV_CAP_DEV_STATUS,
> + CXL_DEV_CAP_MAILBOX_PRIMARY,
> + CXL_DEV_CAP_MAILBOX_SECONDARY,
Does the OS ever uses the SECONDARY mailbox?
> + CXL_DEV_CAP_MEMDEV,
> +};
> +
> struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
>
> void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port
2024-09-11 22:17 ` Dave Jiang
@ 2024-09-16 8:36 ` Alejandro Lucero Palau
2024-09-16 16:07 ` Dave Jiang
0 siblings, 1 reply; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 8:36 UTC (permalink / raw)
To: Dave Jiang, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/11/24 23:17, Dave Jiang wrote:
>
> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Type2 devices have some Type3 functionalities as optional like an mbox
>> or an hdm decoder, and CXL core needs a way to know what an CXL accelerator
>> implements.
>>
>> Add a new field for keeping device capabilities as discovered during
>> initialization.
>>
>> Add same field to cxl_port which for an endpoint will use those
>> capabilities discovered previously, and which will be initialized when
>> calling cxl_port_setup_regs for no endpoints.
> I don't quite understand what you are trying to say here.
I guess you mean the last paragraph, don't you?
If so, the point is the cxl_setup_regs or the register discovery is also
being used from the cxl port code, I think for CXL switches initialization.
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/port.c | 9 +++++----
>> drivers/cxl/core/regs.c | 20 +++++++++++++-------
>> drivers/cxl/cxl.h | 8 +++++---
>> drivers/cxl/cxlmem.h | 2 ++
>> drivers/cxl/pci.c | 9 +++++----
>> include/linux/cxl/cxl.h | 30 ++++++++++++++++++++++++++++++
>> 6 files changed, 60 insertions(+), 18 deletions(-)
>>
>> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
>> index 1d5007e3795a..39b20ddd0296 100644
>> --- a/drivers/cxl/core/port.c
>> +++ b/drivers/cxl/core/port.c
>> @@ -749,7 +749,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
>> }
>>
>> static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
>> - resource_size_t component_reg_phys)
>> + resource_size_t component_reg_phys, u32 *caps)
>> {
>> *map = (struct cxl_register_map) {
>> .host = host,
>> @@ -763,7 +763,7 @@ static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map
>> map->reg_type = CXL_REGLOC_RBI_COMPONENT;
>> map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
>>
>> - return cxl_setup_regs(map);
>> + return cxl_setup_regs(map, caps);
>> }
>>
>> static int cxl_port_setup_regs(struct cxl_port *port,
>> @@ -772,7 +772,7 @@ static int cxl_port_setup_regs(struct cxl_port *port,
>> if (dev_is_platform(port->uport_dev))
>> return 0;
>> return cxl_setup_comp_regs(&port->dev, &port->reg_map,
>> - component_reg_phys);
>> + component_reg_phys, &port->capabilities);
>> }
>>
>> static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
>> @@ -789,7 +789,7 @@ static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
>> * NULL.
>> */
>> rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map,
>> - component_reg_phys);
>> + component_reg_phys, &dport->port->capabilities);
>> dport->reg_map.host = host;
>> return rc;
>> }
>> @@ -858,6 +858,7 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host,
>> port->reg_map = cxlds->reg_map;
>> port->reg_map.host = &port->dev;
>> cxlmd->endpoint = port;
>> + port->capabilities = cxlds->capabilities;
>> } else if (parent_dport) {
>> rc = dev_set_name(dev, "port%d", port->id);
>> if (rc)
>> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
>> index e1082e749c69..8b8abcadcb93 100644
>> --- a/drivers/cxl/core/regs.c
>> +++ b/drivers/cxl/core/regs.c
>> @@ -1,6 +1,7 @@
>> // SPDX-License-Identifier: GPL-2.0-only
>> /* Copyright(c) 2020 Intel Corporation. */
>> #include <linux/io-64-nonatomic-lo-hi.h>
>> +#include <linux/cxl/cxl.h>
>> #include <linux/device.h>
>> #include <linux/slab.h>
>> #include <linux/pci.h>
>> @@ -36,7 +37,7 @@
>> * Probe for component register information and return it in map object.
>> */
>> void cxl_probe_component_regs(struct device *dev, void __iomem *base,
>> - struct cxl_component_reg_map *map)
>> + struct cxl_component_reg_map *map, u32 *caps)
>> {
>> int cap, cap_count;
>> u32 cap_array;
>> @@ -84,6 +85,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
>> decoder_cnt = cxl_hdm_decoder_count(hdr);
>> length = 0x20 * decoder_cnt + 0x10;
>> rmap = &map->hdm_decoder;
>> + *caps |= BIT(CXL_DEV_CAP_HDM);
>> break;
>> }
>> case CXL_CM_CAP_CAP_ID_RAS:
>> @@ -91,6 +93,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
>> offset);
>> length = CXL_RAS_CAPABILITY_LENGTH;
>> rmap = &map->ras;
>> + *caps |= BIT(CXL_DEV_CAP_RAS);
>> break;
>> default:
>> dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id,
>> @@ -117,7 +120,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_probe_component_regs, CXL);
>> * Probe for device register information and return it in map object.
>> */
>> void cxl_probe_device_regs(struct device *dev, void __iomem *base,
>> - struct cxl_device_reg_map *map)
>> + struct cxl_device_reg_map *map, u32 *caps)
>> {
>> int cap, cap_count;
>> u64 cap_array;
>> @@ -146,10 +149,12 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base,
>> case CXLDEV_CAP_CAP_ID_DEVICE_STATUS:
>> dev_dbg(dev, "found Status capability (0x%x)\n", offset);
>> rmap = &map->status;
>> + *caps |= BIT(CXL_DEV_CAP_DEV_STATUS);
>> break;
>> case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX:
>> dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset);
>> rmap = &map->mbox;
>> + *caps |= BIT(CXL_DEV_CAP_MAILBOX_PRIMARY);
>> break;
>> case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX:
>> dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset);
>> @@ -157,6 +162,7 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base,
>> case CXLDEV_CAP_CAP_ID_MEMDEV:
>> dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset);
>> rmap = &map->memdev;
>> + *caps |= BIT(CXL_DEV_CAP_MEMDEV);
>> break;
>> default:
>> if (cap_id >= 0x8000)
>> @@ -421,7 +427,7 @@ static void cxl_unmap_regblock(struct cxl_register_map *map)
>> map->base = NULL;
>> }
>>
>> -static int cxl_probe_regs(struct cxl_register_map *map)
>> +static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
>> {
>> struct cxl_component_reg_map *comp_map;
>> struct cxl_device_reg_map *dev_map;
>> @@ -431,12 +437,12 @@ static int cxl_probe_regs(struct cxl_register_map *map)
>> switch (map->reg_type) {
>> case CXL_REGLOC_RBI_COMPONENT:
>> comp_map = &map->component_map;
>> - cxl_probe_component_regs(host, base, comp_map);
>> + cxl_probe_component_regs(host, base, comp_map, caps);
>> dev_dbg(host, "Set up component registers\n");
>> break;
>> case CXL_REGLOC_RBI_MEMDEV:
>> dev_map = &map->device_map;
>> - cxl_probe_device_regs(host, base, dev_map);
>> + cxl_probe_device_regs(host, base, dev_map, caps);
>> if (!dev_map->status.valid || !dev_map->mbox.valid ||
>> !dev_map->memdev.valid) {
>> dev_err(host, "registers not found: %s%s%s\n",
>> @@ -455,7 +461,7 @@ static int cxl_probe_regs(struct cxl_register_map *map)
>> return 0;
>> }
>>
>> -int cxl_setup_regs(struct cxl_register_map *map)
>> +int cxl_setup_regs(struct cxl_register_map *map, u32 *caps)
>> {
>> int rc;
>>
>> @@ -463,7 +469,7 @@ int cxl_setup_regs(struct cxl_register_map *map)
>> if (rc)
>> return rc;
>>
>> - rc = cxl_probe_regs(map);
>> + rc = cxl_probe_regs(map, caps);
>> cxl_unmap_regblock(map);
>>
>> return rc;
>> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
>> index 9afb407d438f..07c153aa3d77 100644
>> --- a/drivers/cxl/cxl.h
>> +++ b/drivers/cxl/cxl.h
>> @@ -284,9 +284,9 @@ struct cxl_register_map {
>> };
>>
>> void cxl_probe_component_regs(struct device *dev, void __iomem *base,
>> - struct cxl_component_reg_map *map);
>> + struct cxl_component_reg_map *map, u32 *caps);
>> void cxl_probe_device_regs(struct device *dev, void __iomem *base,
>> - struct cxl_device_reg_map *map);
>> + struct cxl_device_reg_map *map, u32 *caps);
>> int cxl_map_component_regs(const struct cxl_register_map *map,
>> struct cxl_component_regs *regs,
>> unsigned long map_mask);
>> @@ -300,7 +300,7 @@ int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type,
>> struct cxl_register_map *map, int index);
>> int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
>> struct cxl_register_map *map);
>> -int cxl_setup_regs(struct cxl_register_map *map);
>> +int cxl_setup_regs(struct cxl_register_map *map, u32 *caps);
>> struct cxl_dport;
>> resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
>> struct cxl_dport *dport);
>> @@ -600,6 +600,7 @@ struct cxl_dax_region {
>> * @cdat: Cached CDAT data
>> * @cdat_available: Should a CDAT attribute be available in sysfs
>> * @pci_latency: Upstream latency in picoseconds
>> + * @capabilities: those capabilities as defined in device mapped registers
>> */
>> struct cxl_port {
>> struct device dev;
>> @@ -623,6 +624,7 @@ struct cxl_port {
>> } cdat;
>> bool cdat_available;
>> long pci_latency;
>> + u32 capabilities;
>> };
>>
>> /**
>> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
>> index afb53d058d62..37c043100300 100644
>> --- a/drivers/cxl/cxlmem.h
>> +++ b/drivers/cxl/cxlmem.h
>> @@ -424,6 +424,7 @@ struct cxl_dpa_perf {
>> * @ram_res: Active Volatile memory capacity configuration
>> * @serial: PCIe Device Serial Number
>> * @type: Generic Memory Class device or Vendor Specific Memory device
>> + * @capabilities: those capabilities as defined in device mapped registers
>> */
>> struct cxl_dev_state {
>> struct device *dev;
>> @@ -438,6 +439,7 @@ struct cxl_dev_state {
>> struct resource ram_res;
>> u64 serial;
>> enum cxl_devtype type;
>> + u32 capabilities;
>> };
>>
>> /**
>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>> index 742a7b2a1be5..58f325019886 100644
>> --- a/drivers/cxl/pci.c
>> +++ b/drivers/cxl/pci.c
>> @@ -503,7 +503,7 @@ static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
>> }
>>
>> static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
>> - struct cxl_register_map *map)
>> + struct cxl_register_map *map, u32 *caps)
>> {
>> int rc;
>>
>> @@ -520,7 +520,7 @@ static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
>> if (rc)
>> return rc;
>>
>> - return cxl_setup_regs(map);
>> + return cxl_setup_regs(map, caps);
>> }
>>
>> static int cxl_pci_ras_unmask(struct pci_dev *pdev)
>> @@ -827,7 +827,8 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> else
>> cxl_set_dvsec(cxlds, dvsec);
>>
>> - rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
>> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map,
>> + &cxlds->capabilities);
>> if (rc)
>> return rc;
>>
>> @@ -840,7 +841,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> * still be useful for management functions so don't return an error.
>> */
>> rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
>> - &cxlds->reg_map);
>> + &cxlds->reg_map, &cxlds->capabilities);
>> if (rc)
>> dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
>> else if (!cxlds->reg_map.component_map.ras.valid)
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index e78eefa82123..930b1b9c1d6a 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -12,6 +12,36 @@ enum cxl_resource {
>> CXL_ACCEL_RES_PMEM,
>> };
>>
>> +/* Capabilities as defined for:
>> + *
>> + * Component Registers (Table 8-22 CXL 3.0 specification)
>> + * Device Registers (8.2.8.2.1 CXL 3.0 specification)
> Should just use 3.1 since that's the latest spec.
Ok.
>> + */
>> +
>> +enum cxl_dev_cap {
>> + /* capabilities from Component Registers */
>> + CXL_DEV_CAP_RAS,
>> + CXL_DEV_CAP_SEC,
>> + CXL_DEV_CAP_LINK,
>> + CXL_DEV_CAP_HDM,
>> + CXL_DEV_CAP_SEC_EXT,
>> + CXL_DEV_CAP_IDE,
>> + CXL_DEV_CAP_SNOOP_FILTER,
>> + CXL_DEV_CAP_TIMEOUT_AND_ISOLATION,
>> + CXL_DEV_CAP_CACHEMEM_EXT,
>> + CXL_DEV_CAP_BI_ROUTE_TABLE,
>> + CXL_DEV_CAP_BI_DECODER,
>> + CXL_DEV_CAP_CACHEID_ROUTE_TABLE,
>> + CXL_DEV_CAP_CACHEID_DECODER,
>> + CXL_DEV_CAP_HDM_EXT,
>> + CXL_DEV_CAP_METADATA_EXT,
>> + /* capabilities from Device Registers */
>> + CXL_DEV_CAP_DEV_STATUS,
>> + CXL_DEV_CAP_MAILBOX_PRIMARY,
>> + CXL_DEV_CAP_MAILBOX_SECONDARY,
> Does the OS ever uses the SECONDARY mailbox?
I have no idea. I'm just listing all the potential capabilities here as
you can see for things like BI or SNOOP.
Should I just add those referenced by code?
>> + CXL_DEV_CAP_MEMDEV,
>> +};
>> +
>> struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
>>
>> void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port
2024-09-16 8:36 ` Alejandro Lucero Palau
@ 2024-09-16 16:07 ` Dave Jiang
0 siblings, 0 replies; 88+ messages in thread
From: Dave Jiang @ 2024-09-16 16:07 UTC (permalink / raw)
To: Alejandro Lucero Palau, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/16/24 1:36 AM, Alejandro Lucero Palau wrote:
>
> On 9/11/24 23:17, Dave Jiang wrote:
>>
>> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>>> From: Alejandro Lucero <alucerop@amd.com>
>>>
>>> Type2 devices have some Type3 functionalities as optional like an mbox
>>> or an hdm decoder, and CXL core needs a way to know what an CXL accelerator
>>> implements.
>>>
>>> Add a new field for keeping device capabilities as discovered during
>>> initialization.
>>>
>>> Add same field to cxl_port which for an endpoint will use those
>>> capabilities discovered previously, and which will be initialized when
>>> calling cxl_port_setup_regs for no endpoints.
>> I don't quite understand what you are trying to say here.
>
>
> I guess you mean the last paragraph, don't you?
>
> If so, the point is the cxl_setup_regs or the register discovery is also being used from the cxl port code, I think for CXL switches initialization.
Yes. Your response clarified my confusion. I do suggest you say that in your commit log.
>
>
>>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>>> ---
>>> drivers/cxl/core/port.c | 9 +++++----
>>> drivers/cxl/core/regs.c | 20 +++++++++++++-------
>>> drivers/cxl/cxl.h | 8 +++++---
>>> drivers/cxl/cxlmem.h | 2 ++
>>> drivers/cxl/pci.c | 9 +++++----
>>> include/linux/cxl/cxl.h | 30 ++++++++++++++++++++++++++++++
>>> 6 files changed, 60 insertions(+), 18 deletions(-)
>>>
>>> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
>>> index 1d5007e3795a..39b20ddd0296 100644
>>> --- a/drivers/cxl/core/port.c
>>> +++ b/drivers/cxl/core/port.c
>>> @@ -749,7 +749,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
>>> }
>>> static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
>>> - resource_size_t component_reg_phys)
>>> + resource_size_t component_reg_phys, u32 *caps)
>>> {
>>> *map = (struct cxl_register_map) {
>>> .host = host,
>>> @@ -763,7 +763,7 @@ static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map
>>> map->reg_type = CXL_REGLOC_RBI_COMPONENT;
>>> map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
>>> - return cxl_setup_regs(map);
>>> + return cxl_setup_regs(map, caps);
>>> }
>>> static int cxl_port_setup_regs(struct cxl_port *port,
>>> @@ -772,7 +772,7 @@ static int cxl_port_setup_regs(struct cxl_port *port,
>>> if (dev_is_platform(port->uport_dev))
>>> return 0;
>>> return cxl_setup_comp_regs(&port->dev, &port->reg_map,
>>> - component_reg_phys);
>>> + component_reg_phys, &port->capabilities);
>>> }
>>> static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
>>> @@ -789,7 +789,7 @@ static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
>>> * NULL.
>>> */
>>> rc = cxl_setup_comp_regs(dport->dport_dev, &dport->reg_map,
>>> - component_reg_phys);
>>> + component_reg_phys, &dport->port->capabilities);
>>> dport->reg_map.host = host;
>>> return rc;
>>> }
>>> @@ -858,6 +858,7 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host,
>>> port->reg_map = cxlds->reg_map;
>>> port->reg_map.host = &port->dev;
>>> cxlmd->endpoint = port;
>>> + port->capabilities = cxlds->capabilities;
>>> } else if (parent_dport) {
>>> rc = dev_set_name(dev, "port%d", port->id);
>>> if (rc)
>>> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
>>> index e1082e749c69..8b8abcadcb93 100644
>>> --- a/drivers/cxl/core/regs.c
>>> +++ b/drivers/cxl/core/regs.c
>>> @@ -1,6 +1,7 @@
>>> // SPDX-License-Identifier: GPL-2.0-only
>>> /* Copyright(c) 2020 Intel Corporation. */
>>> #include <linux/io-64-nonatomic-lo-hi.h>
>>> +#include <linux/cxl/cxl.h>
>>> #include <linux/device.h>
>>> #include <linux/slab.h>
>>> #include <linux/pci.h>
>>> @@ -36,7 +37,7 @@
>>> * Probe for component register information and return it in map object.
>>> */
>>> void cxl_probe_component_regs(struct device *dev, void __iomem *base,
>>> - struct cxl_component_reg_map *map)
>>> + struct cxl_component_reg_map *map, u32 *caps)
>>> {
>>> int cap, cap_count;
>>> u32 cap_array;
>>> @@ -84,6 +85,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
>>> decoder_cnt = cxl_hdm_decoder_count(hdr);
>>> length = 0x20 * decoder_cnt + 0x10;
>>> rmap = &map->hdm_decoder;
>>> + *caps |= BIT(CXL_DEV_CAP_HDM);
>>> break;
>>> }
>>> case CXL_CM_CAP_CAP_ID_RAS:
>>> @@ -91,6 +93,7 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
>>> offset);
>>> length = CXL_RAS_CAPABILITY_LENGTH;
>>> rmap = &map->ras;
>>> + *caps |= BIT(CXL_DEV_CAP_RAS);
>>> break;
>>> default:
>>> dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id,
>>> @@ -117,7 +120,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_probe_component_regs, CXL);
>>> * Probe for device register information and return it in map object.
>>> */
>>> void cxl_probe_device_regs(struct device *dev, void __iomem *base,
>>> - struct cxl_device_reg_map *map)
>>> + struct cxl_device_reg_map *map, u32 *caps)
>>> {
>>> int cap, cap_count;
>>> u64 cap_array;
>>> @@ -146,10 +149,12 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base,
>>> case CXLDEV_CAP_CAP_ID_DEVICE_STATUS:
>>> dev_dbg(dev, "found Status capability (0x%x)\n", offset);
>>> rmap = &map->status;
>>> + *caps |= BIT(CXL_DEV_CAP_DEV_STATUS);
>>> break;
>>> case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX:
>>> dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset);
>>> rmap = &map->mbox;
>>> + *caps |= BIT(CXL_DEV_CAP_MAILBOX_PRIMARY);
>>> break;
>>> case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX:
>>> dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset);
>>> @@ -157,6 +162,7 @@ void cxl_probe_device_regs(struct device *dev, void __iomem *base,
>>> case CXLDEV_CAP_CAP_ID_MEMDEV:
>>> dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset);
>>> rmap = &map->memdev;
>>> + *caps |= BIT(CXL_DEV_CAP_MEMDEV);
>>> break;
>>> default:
>>> if (cap_id >= 0x8000)
>>> @@ -421,7 +427,7 @@ static void cxl_unmap_regblock(struct cxl_register_map *map)
>>> map->base = NULL;
>>> }
>>> -static int cxl_probe_regs(struct cxl_register_map *map)
>>> +static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
>>> {
>>> struct cxl_component_reg_map *comp_map;
>>> struct cxl_device_reg_map *dev_map;
>>> @@ -431,12 +437,12 @@ static int cxl_probe_regs(struct cxl_register_map *map)
>>> switch (map->reg_type) {
>>> case CXL_REGLOC_RBI_COMPONENT:
>>> comp_map = &map->component_map;
>>> - cxl_probe_component_regs(host, base, comp_map);
>>> + cxl_probe_component_regs(host, base, comp_map, caps);
>>> dev_dbg(host, "Set up component registers\n");
>>> break;
>>> case CXL_REGLOC_RBI_MEMDEV:
>>> dev_map = &map->device_map;
>>> - cxl_probe_device_regs(host, base, dev_map);
>>> + cxl_probe_device_regs(host, base, dev_map, caps);
>>> if (!dev_map->status.valid || !dev_map->mbox.valid ||
>>> !dev_map->memdev.valid) {
>>> dev_err(host, "registers not found: %s%s%s\n",
>>> @@ -455,7 +461,7 @@ static int cxl_probe_regs(struct cxl_register_map *map)
>>> return 0;
>>> }
>>> -int cxl_setup_regs(struct cxl_register_map *map)
>>> +int cxl_setup_regs(struct cxl_register_map *map, u32 *caps)
>>> {
>>> int rc;
>>> @@ -463,7 +469,7 @@ int cxl_setup_regs(struct cxl_register_map *map)
>>> if (rc)
>>> return rc;
>>> - rc = cxl_probe_regs(map);
>>> + rc = cxl_probe_regs(map, caps);
>>> cxl_unmap_regblock(map);
>>> return rc;
>>> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
>>> index 9afb407d438f..07c153aa3d77 100644
>>> --- a/drivers/cxl/cxl.h
>>> +++ b/drivers/cxl/cxl.h
>>> @@ -284,9 +284,9 @@ struct cxl_register_map {
>>> };
>>> void cxl_probe_component_regs(struct device *dev, void __iomem *base,
>>> - struct cxl_component_reg_map *map);
>>> + struct cxl_component_reg_map *map, u32 *caps);
>>> void cxl_probe_device_regs(struct device *dev, void __iomem *base,
>>> - struct cxl_device_reg_map *map);
>>> + struct cxl_device_reg_map *map, u32 *caps);
>>> int cxl_map_component_regs(const struct cxl_register_map *map,
>>> struct cxl_component_regs *regs,
>>> unsigned long map_mask);
>>> @@ -300,7 +300,7 @@ int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type,
>>> struct cxl_register_map *map, int index);
>>> int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
>>> struct cxl_register_map *map);
>>> -int cxl_setup_regs(struct cxl_register_map *map);
>>> +int cxl_setup_regs(struct cxl_register_map *map, u32 *caps);
>>> struct cxl_dport;
>>> resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
>>> struct cxl_dport *dport);
>>> @@ -600,6 +600,7 @@ struct cxl_dax_region {
>>> * @cdat: Cached CDAT data
>>> * @cdat_available: Should a CDAT attribute be available in sysfs
>>> * @pci_latency: Upstream latency in picoseconds
>>> + * @capabilities: those capabilities as defined in device mapped registers
>>> */
>>> struct cxl_port {
>>> struct device dev;
>>> @@ -623,6 +624,7 @@ struct cxl_port {
>>> } cdat;
>>> bool cdat_available;
>>> long pci_latency;
>>> + u32 capabilities;
>>> };
>>> /**
>>> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
>>> index afb53d058d62..37c043100300 100644
>>> --- a/drivers/cxl/cxlmem.h
>>> +++ b/drivers/cxl/cxlmem.h
>>> @@ -424,6 +424,7 @@ struct cxl_dpa_perf {
>>> * @ram_res: Active Volatile memory capacity configuration
>>> * @serial: PCIe Device Serial Number
>>> * @type: Generic Memory Class device or Vendor Specific Memory device
>>> + * @capabilities: those capabilities as defined in device mapped registers
>>> */
>>> struct cxl_dev_state {
>>> struct device *dev;
>>> @@ -438,6 +439,7 @@ struct cxl_dev_state {
>>> struct resource ram_res;
>>> u64 serial;
>>> enum cxl_devtype type;
>>> + u32 capabilities;
>>> };
>>> /**
>>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>>> index 742a7b2a1be5..58f325019886 100644
>>> --- a/drivers/cxl/pci.c
>>> +++ b/drivers/cxl/pci.c
>>> @@ -503,7 +503,7 @@ static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
>>> }
>>> static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
>>> - struct cxl_register_map *map)
>>> + struct cxl_register_map *map, u32 *caps)
>>> {
>>> int rc;
>>> @@ -520,7 +520,7 @@ static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
>>> if (rc)
>>> return rc;
>>> - return cxl_setup_regs(map);
>>> + return cxl_setup_regs(map, caps);
>>> }
>>> static int cxl_pci_ras_unmask(struct pci_dev *pdev)
>>> @@ -827,7 +827,8 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>>> else
>>> cxl_set_dvsec(cxlds, dvsec);
>>> - rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map);
>>> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map,
>>> + &cxlds->capabilities);
>>> if (rc)
>>> return rc;
>>> @@ -840,7 +841,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>>> * still be useful for management functions so don't return an error.
>>> */
>>> rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
>>> - &cxlds->reg_map);
>>> + &cxlds->reg_map, &cxlds->capabilities);
>>> if (rc)
>>> dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
>>> else if (!cxlds->reg_map.component_map.ras.valid)
>>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>>> index e78eefa82123..930b1b9c1d6a 100644
>>> --- a/include/linux/cxl/cxl.h
>>> +++ b/include/linux/cxl/cxl.h
>>> @@ -12,6 +12,36 @@ enum cxl_resource {
>>> CXL_ACCEL_RES_PMEM,
>>> };
>>> +/* Capabilities as defined for:
>>> + *
>>> + * Component Registers (Table 8-22 CXL 3.0 specification)
>>> + * Device Registers (8.2.8.2.1 CXL 3.0 specification)
>> Should just use 3.1 since that's the latest spec.
>
>
> Ok.
>
>
>>> + */
>>> +
>>> +enum cxl_dev_cap {
>>> + /* capabilities from Component Registers */
>>> + CXL_DEV_CAP_RAS,
>>> + CXL_DEV_CAP_SEC,
>>> + CXL_DEV_CAP_LINK,
>>> + CXL_DEV_CAP_HDM,
>>> + CXL_DEV_CAP_SEC_EXT,
>>> + CXL_DEV_CAP_IDE,
>>> + CXL_DEV_CAP_SNOOP_FILTER,
>>> + CXL_DEV_CAP_TIMEOUT_AND_ISOLATION,
>>> + CXL_DEV_CAP_CACHEMEM_EXT,
>>> + CXL_DEV_CAP_BI_ROUTE_TABLE,
>>> + CXL_DEV_CAP_BI_DECODER,
>>> + CXL_DEV_CAP_CACHEID_ROUTE_TABLE,
>>> + CXL_DEV_CAP_CACHEID_DECODER,
>>> + CXL_DEV_CAP_HDM_EXT,
>>> + CXL_DEV_CAP_METADATA_EXT,
>>> + /* capabilities from Device Registers */
>>> + CXL_DEV_CAP_DEV_STATUS,
>>> + CXL_DEV_CAP_MAILBOX_PRIMARY,
>>> + CXL_DEV_CAP_MAILBOX_SECONDARY,
>> Does the OS ever uses the SECONDARY mailbox?
>
>
> I have no idea. I'm just listing all the potential capabilities here as you can see for things like BI or SNOOP.
>
> Should I just add those referenced by code?
>
>
>>> + CXL_DEV_CAP_MEMDEV,
>>> +};
>>> +
>>> struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
>>> void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port
2024-09-07 8:18 ` [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port alejandro.lucero-palau
2024-09-07 18:08 ` kernel test robot
2024-09-11 22:17 ` Dave Jiang
@ 2024-09-13 17:25 ` Jonathan Cameron
2024-09-16 12:13 ` Alejandro Lucero Palau
2 siblings, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 17:25 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:18 +0100
alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Type2 devices have some Type3 functionalities as optional like an mbox
> or an hdm decoder, and CXL core needs a way to know what an CXL accelerator
> implements.
>
> Add a new field for keeping device capabilities as discovered during
> initialization.
>
> Add same field to cxl_port which for an endpoint will use those
> capabilities discovered previously, and which will be initialized when
> calling cxl_port_setup_regs for no endpoints.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
Hi,
My only real suggestion on this one is to use a bitmap to make
it easy to extend the capabilities as needed in future.
That means passing an unsigned long pointer around.
> @@ -600,6 +600,7 @@ struct cxl_dax_region {
> * @cdat: Cached CDAT data
> * @cdat_available: Should a CDAT attribute be available in sysfs
> * @pci_latency: Upstream latency in picoseconds
> + * @capabilities: those capabilities as defined in device mapped registers
> */
> struct cxl_port {
> struct device dev;
> @@ -623,6 +624,7 @@ struct cxl_port {
> } cdat;
> bool cdat_available;
> long pci_latency;
> + u32 capabilities;
Use DECLARE_BITMAP() for this to make life easy should we ever
have more than 32.
> };
>
> /**
> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> index afb53d058d62..37c043100300 100644
> --- a/drivers/cxl/cxlmem.h
> +++ b/drivers/cxl/cxlmem.h
> @@ -424,6 +424,7 @@ struct cxl_dpa_perf {
> * @ram_res: Active Volatile memory capacity configuration
> * @serial: PCIe Device Serial Number
> * @type: Generic Memory Class device or Vendor Specific Memory device
> + * @capabilities: those capabilities as defined in device mapped registers
> */
> struct cxl_dev_state {
> struct device *dev;
> @@ -438,6 +439,7 @@ struct cxl_dev_state {
> struct resource ram_res;
> u64 serial;
> enum cxl_devtype type;
> + u32 capabilities;
As above, use a bitmap and access it with the various bitmap operators
so that we aren't constrained to 32 bits given half are
used already.
> };
>
> /**
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index e78eefa82123..930b1b9c1d6a 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -12,6 +12,36 @@ enum cxl_resource {
> CXL_ACCEL_RES_PMEM,
> };
>
> +/* Capabilities as defined for:
Trivial but cxl tends to do
/*
* Capabilities ..
style multiline comments.
> + *
> + * Component Registers (Table 8-22 CXL 3.0 specification)
> + * Device Registers (8.2.8.2.1 CXL 3.0 specification)
> + */
> +
> +enum cxl_dev_cap {
> + /* capabilities from Component Registers */
> + CXL_DEV_CAP_RAS,
> + CXL_DEV_CAP_SEC,
> + CXL_DEV_CAP_LINK,
> + CXL_DEV_CAP_HDM,
> + CXL_DEV_CAP_SEC_EXT,
> + CXL_DEV_CAP_IDE,
> + CXL_DEV_CAP_SNOOP_FILTER,
> + CXL_DEV_CAP_TIMEOUT_AND_ISOLATION,
> + CXL_DEV_CAP_CACHEMEM_EXT,
> + CXL_DEV_CAP_BI_ROUTE_TABLE,
> + CXL_DEV_CAP_BI_DECODER,
> + CXL_DEV_CAP_CACHEID_ROUTE_TABLE,
> + CXL_DEV_CAP_CACHEID_DECODER,
> + CXL_DEV_CAP_HDM_EXT,
> + CXL_DEV_CAP_METADATA_EXT,
> + /* capabilities from Device Registers */
> + CXL_DEV_CAP_DEV_STATUS,
> + CXL_DEV_CAP_MAILBOX_PRIMARY,
> + CXL_DEV_CAP_MAILBOX_SECONDARY,
Dan raised this one already - definitely not something any
driver in Linux should be aware of. Hence just drop this entry.
> + CXL_DEV_CAP_MEMDEV,
> +};
> +
> struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
>
> void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port
2024-09-13 17:25 ` Jonathan Cameron
@ 2024-09-16 12:13 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 12:13 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/13/24 18:25, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:18 +0100
> alejandro.lucero-palau@amd.com wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Type2 devices have some Type3 functionalities as optional like an mbox
>> or an hdm decoder, and CXL core needs a way to know what an CXL accelerator
>> implements.
>>
>> Add a new field for keeping device capabilities as discovered during
>> initialization.
>>
>> Add same field to cxl_port which for an endpoint will use those
>> capabilities discovered previously, and which will be initialized when
>> calling cxl_port_setup_regs for no endpoints.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Hi,
>
> My only real suggestion on this one is to use a bitmap to make
> it easy to extend the capabilities as needed in future.
> That means passing an unsigned long pointer around.
>
It makes sense.
I'll do.
>> @@ -600,6 +600,7 @@ struct cxl_dax_region {
>> * @cdat: Cached CDAT data
>> * @cdat_available: Should a CDAT attribute be available in sysfs
>> * @pci_latency: Upstream latency in picoseconds
>> + * @capabilities: those capabilities as defined in device mapped registers
>> */
>> struct cxl_port {
>> struct device dev;
>> @@ -623,6 +624,7 @@ struct cxl_port {
>> } cdat;
>> bool cdat_available;
>> long pci_latency;
>> + u32 capabilities;
> Use DECLARE_BITMAP() for this to make life easy should we ever
> have more than 32.
>
>> };
>>
>> /**
>> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
>> index afb53d058d62..37c043100300 100644
>> --- a/drivers/cxl/cxlmem.h
>> +++ b/drivers/cxl/cxlmem.h
>> @@ -424,6 +424,7 @@ struct cxl_dpa_perf {
>> * @ram_res: Active Volatile memory capacity configuration
>> * @serial: PCIe Device Serial Number
>> * @type: Generic Memory Class device or Vendor Specific Memory device
>> + * @capabilities: those capabilities as defined in device mapped registers
>> */
>> struct cxl_dev_state {
>> struct device *dev;
>> @@ -438,6 +439,7 @@ struct cxl_dev_state {
>> struct resource ram_res;
>> u64 serial;
>> enum cxl_devtype type;
>> + u32 capabilities;
> As above, use a bitmap and access it with the various bitmap operators
> so that we aren't constrained to 32 bits given half are
> used already.
I though maybe I should use u64 ...
>
>> };
>>
>> /**
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index e78eefa82123..930b1b9c1d6a 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -12,6 +12,36 @@ enum cxl_resource {
>> CXL_ACCEL_RES_PMEM,
>> };
>>
>> +/* Capabilities as defined for:
> Trivial but cxl tends to do
> /*
> * Capabilities ..
>
> style multiline comments.
Right. I'll fit it.
>> + *
>> + * Component Registers (Table 8-22 CXL 3.0 specification)
>> + * Device Registers (8.2.8.2.1 CXL 3.0 specification)
>> + */
>> +
>> +enum cxl_dev_cap {
>> + /* capabilities from Component Registers */
>> + CXL_DEV_CAP_RAS,
>> + CXL_DEV_CAP_SEC,
>> + CXL_DEV_CAP_LINK,
>> + CXL_DEV_CAP_HDM,
>> + CXL_DEV_CAP_SEC_EXT,
>> + CXL_DEV_CAP_IDE,
>> + CXL_DEV_CAP_SNOOP_FILTER,
>> + CXL_DEV_CAP_TIMEOUT_AND_ISOLATION,
>> + CXL_DEV_CAP_CACHEMEM_EXT,
>> + CXL_DEV_CAP_BI_ROUTE_TABLE,
>> + CXL_DEV_CAP_BI_DECODER,
>> + CXL_DEV_CAP_CACHEID_ROUTE_TABLE,
>> + CXL_DEV_CAP_CACHEID_DECODER,
>> + CXL_DEV_CAP_HDM_EXT,
>> + CXL_DEV_CAP_METADATA_EXT,
>> + /* capabilities from Device Registers */
>> + CXL_DEV_CAP_DEV_STATUS,
>> + CXL_DEV_CAP_MAILBOX_PRIMARY,
>> + CXL_DEV_CAP_MAILBOX_SECONDARY,
> Dan raised this one already - definitely not something any
> driver in Linux should be aware of. Hence just drop this entry.
You mean never ever or just by now?
Maybe I'm missing something specific to the secondary mailbox, but if
the rule is if none used yet, do not add it, then there are other caps I
should remove as well.
>> + CXL_DEV_CAP_MEMDEV,
>> +};
>> +
>> struct cxl_dev_state *cxl_accel_state_create(struct device *dev);
>>
>> void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 03/20] cxl/pci: add check for validating capabilities
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
2024-09-07 8:18 ` [PATCH v3 01/20] cxl: add type2 device basic support alejandro.lucero-palau
2024-09-07 8:18 ` [PATCH v3 02/20] cxl: add capabilities field to cxl_dev_state and cxl_port alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-10 3:26 ` Li, Ming4
` (2 more replies)
2024-09-07 8:18 ` [PATCH v3 04/20] cxl: move pci generic code alejandro.lucero-palau
` (16 subsequent siblings)
19 siblings, 3 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
During CXL device initialization supported capabilities by the device
are discovered. Type3 and Type2 devices have different mandatory
capabilities and a Type2 expects a specific set including optional
capabilities.
Add a function for checking expected capabilities against those found
during initialization.
Rely on this function for validating capabilities instead of when CXL
regs are probed.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/pci.c | 17 +++++++++++++++++
drivers/cxl/core/regs.c | 9 ---------
drivers/cxl/pci.c | 12 ++++++++++++
include/linux/cxl/cxl.h | 2 ++
4 files changed, 31 insertions(+), 9 deletions(-)
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index 3d6564dbda57..57370d9beb32 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -7,6 +7,7 @@
#include <linux/pci.h>
#include <linux/pci-doe.h>
#include <linux/aer.h>
+#include <linux/cxl/cxl.h>
#include <linux/cxl/pci.h>
#include <cxlpci.h>
#include <cxlmem.h>
@@ -1077,3 +1078,19 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
__cxl_endpoint_decoder_reset_detected);
}
EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
+
+bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
+ u32 *current_caps)
+{
+ if (current_caps)
+ *current_caps = cxlds->capabilities;
+
+ dev_dbg(cxlds->dev, "Checking cxlds caps 0x%08x vs expected caps 0x%08x\n",
+ cxlds->capabilities, expected_caps);
+
+ if ((cxlds->capabilities & expected_caps) != expected_caps)
+ return false;
+
+ return true;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_pci_check_caps, CXL);
diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
index 8b8abcadcb93..35f6dc97be6e 100644
--- a/drivers/cxl/core/regs.c
+++ b/drivers/cxl/core/regs.c
@@ -443,15 +443,6 @@ static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
case CXL_REGLOC_RBI_MEMDEV:
dev_map = &map->device_map;
cxl_probe_device_regs(host, base, dev_map, caps);
- if (!dev_map->status.valid || !dev_map->mbox.valid ||
- !dev_map->memdev.valid) {
- dev_err(host, "registers not found: %s%s%s\n",
- !dev_map->status.valid ? "status " : "",
- !dev_map->mbox.valid ? "mbox " : "",
- !dev_map->memdev.valid ? "memdev " : "");
- return -ENXIO;
- }
-
dev_dbg(host, "Probing device registers...\n");
break;
default:
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index 58f325019886..bec660357eec 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -796,6 +796,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
struct cxl_register_map map;
struct cxl_memdev *cxlmd;
int i, rc, pmu_count;
+ u32 expected, found;
bool irq_avail;
u16 dvsec;
@@ -852,6 +853,17 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (rc)
dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
+ /* These are the mandatory capabilities for a Type3 device */
+ expected = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
+ BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV);
+
+ if (!cxl_pci_check_caps(cxlds, expected, &found)) {
+ dev_err(&pdev->dev,
+ "Expected capabilities not matching with found capabilities: (%08x - %08x)\n",
+ expected, found);
+ return -ENXIO;
+ }
+
rc = cxl_await_media_ready(cxlds);
if (rc == 0)
cxlds->media_ready = true;
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index 930b1b9c1d6a..4a57bf60403d 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -48,4 +48,6 @@ void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
enum cxl_resource);
+bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
+ u32 *current_caps);
#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 03/20] cxl/pci: add check for validating capabilities
2024-09-07 8:18 ` [PATCH v3 03/20] cxl/pci: add check for validating capabilities alejandro.lucero-palau
@ 2024-09-10 3:26 ` Li, Ming4
2024-09-10 6:24 ` Li, Ming4
2024-09-11 23:06 ` Dave Jiang
2024-09-13 17:28 ` Jonathan Cameron
2 siblings, 1 reply; 88+ messages in thread
From: Li, Ming4 @ 2024-09-10 3:26 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> During CXL device initialization supported capabilities by the device
> are discovered. Type3 and Type2 devices have different mandatory
> capabilities and a Type2 expects a specific set including optional
> capabilities.
>
> Add a function for checking expected capabilities against those found
> during initialization.
>
> Rely on this function for validating capabilities instead of when CXL
> regs are probed.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/pci.c | 17 +++++++++++++++++
> drivers/cxl/core/regs.c | 9 ---------
> drivers/cxl/pci.c | 12 ++++++++++++
> include/linux/cxl/cxl.h | 2 ++
> 4 files changed, 31 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> index 3d6564dbda57..57370d9beb32 100644
> --- a/drivers/cxl/core/pci.c
> +++ b/drivers/cxl/core/pci.c
> @@ -7,6 +7,7 @@
> #include <linux/pci.h>
> #include <linux/pci-doe.h>
> #include <linux/aer.h>
> +#include <linux/cxl/cxl.h>
> #include <linux/cxl/pci.h>
> #include <cxlpci.h>
> #include <cxlmem.h>
> @@ -1077,3 +1078,19 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
> __cxl_endpoint_decoder_reset_detected);
> }
> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
> +
> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> + u32 *current_caps)
> +{
> + if (current_caps)
> + *current_caps = cxlds->capabilities;
> +
> + dev_dbg(cxlds->dev, "Checking cxlds caps 0x%08x vs expected caps 0x%08x\n",
> + cxlds->capabilities, expected_caps);
> +
> + if ((cxlds->capabilities & expected_caps) != expected_caps)
> + return false;
> +
> + return true;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_pci_check_caps, CXL);
Why has to use this 'u32 *current_caps' as a parameter? if user wants to know the capabilities of a device, they can get it from cxlds->capabilities directly.
> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
> index 8b8abcadcb93..35f6dc97be6e 100644
> --- a/drivers/cxl/core/regs.c
> +++ b/drivers/cxl/core/regs.c
> @@ -443,15 +443,6 @@ static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
> case CXL_REGLOC_RBI_MEMDEV:
> dev_map = &map->device_map;
> cxl_probe_device_regs(host, base, dev_map, caps);
> - if (!dev_map->status.valid || !dev_map->mbox.valid ||
> - !dev_map->memdev.valid) {
> - dev_err(host, "registers not found: %s%s%s\n",
> - !dev_map->status.valid ? "status " : "",
> - !dev_map->mbox.valid ? "mbox " : "",
> - !dev_map->memdev.valid ? "memdev " : "");
> - return -ENXIO;
> - }
> -
> dev_dbg(host, "Probing device registers...\n");
> break;
> default:
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 58f325019886..bec660357eec 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -796,6 +796,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> struct cxl_register_map map;
> struct cxl_memdev *cxlmd;
> int i, rc, pmu_count;
> + u32 expected, found;
> bool irq_avail;
> u16 dvsec;
>
> @@ -852,6 +853,17 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> if (rc)
> dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
>
> + /* These are the mandatory capabilities for a Type3 device */
> + expected = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
> + BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV);
> +
> + if (!cxl_pci_check_caps(cxlds, expected, &found)) {
> + dev_err(&pdev->dev,
> + "Expected capabilities not matching with found capabilities: (%08x - %08x)\n",
> + expected, found);
> + return -ENXIO;
> + }
> +
Same as above, the capabilities already are cached in cxlds->capabilities. seems like that the 'found' can be removed and using cxlds->capabilities directly here.
> rc = cxl_await_media_ready(cxlds);
> if (rc == 0)
> cxlds->media_ready = true;
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index 930b1b9c1d6a..4a57bf60403d 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -48,4 +48,6 @@ void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
> void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
> int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> enum cxl_resource);
> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> + u32 *current_caps);
> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 03/20] cxl/pci: add check for validating capabilities
2024-09-10 3:26 ` Li, Ming4
@ 2024-09-10 6:24 ` Li, Ming4
2024-09-10 7:31 ` Alejandro Lucero Palau
0 siblings, 1 reply; 88+ messages in thread
From: Li, Ming4 @ 2024-09-10 6:24 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/10/2024 11:26 AM, Li, Ming4 wrote:
> On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> During CXL device initialization supported capabilities by the device
>> are discovered. Type3 and Type2 devices have different mandatory
>> capabilities and a Type2 expects a specific set including optional
>> capabilities.
>>
>> Add a function for checking expected capabilities against those found
>> during initialization.
>>
>> Rely on this function for validating capabilities instead of when CXL
>> regs are probed.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/pci.c | 17 +++++++++++++++++
>> drivers/cxl/core/regs.c | 9 ---------
>> drivers/cxl/pci.c | 12 ++++++++++++
>> include/linux/cxl/cxl.h | 2 ++
>> 4 files changed, 31 insertions(+), 9 deletions(-)
>>
>> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
>> index 3d6564dbda57..57370d9beb32 100644
>> --- a/drivers/cxl/core/pci.c
>> +++ b/drivers/cxl/core/pci.c
>> @@ -7,6 +7,7 @@
>> #include <linux/pci.h>
>> #include <linux/pci-doe.h>
>> #include <linux/aer.h>
>> +#include <linux/cxl/cxl.h>
>> #include <linux/cxl/pci.h>
>> #include <cxlpci.h>
>> #include <cxlmem.h>
>> @@ -1077,3 +1078,19 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
>> __cxl_endpoint_decoder_reset_detected);
>> }
>> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
>> +
>> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> + u32 *current_caps)
>> +{
>> + if (current_caps)
>> + *current_caps = cxlds->capabilities;
>> +
>> + dev_dbg(cxlds->dev, "Checking cxlds caps 0x%08x vs expected caps 0x%08x\n",
>> + cxlds->capabilities, expected_caps);
>> +
>> + if ((cxlds->capabilities & expected_caps) != expected_caps)
>> + return false;
>> +
>> + return true;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_pci_check_caps, CXL);
> Why has to use this 'u32 *current_caps' as a parameter? if user wants to know the capabilities of a device, they can get it from cxlds->capabilities directly.
>
Sorry, I missed something implemented in PATCH #1, seems like you can not access struct cxl_dev_state from efx driver side. right?
>> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
>> index 8b8abcadcb93..35f6dc97be6e 100644
>> --- a/drivers/cxl/core/regs.c
>> +++ b/drivers/cxl/core/regs.c
>> @@ -443,15 +443,6 @@ static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
>> case CXL_REGLOC_RBI_MEMDEV:
>> dev_map = &map->device_map;
>> cxl_probe_device_regs(host, base, dev_map, caps);
>> - if (!dev_map->status.valid || !dev_map->mbox.valid ||
>> - !dev_map->memdev.valid) {
>> - dev_err(host, "registers not found: %s%s%s\n",
>> - !dev_map->status.valid ? "status " : "",
>> - !dev_map->mbox.valid ? "mbox " : "",
>> - !dev_map->memdev.valid ? "memdev " : "");
>> - return -ENXIO;
>> - }
>> -
>> dev_dbg(host, "Probing device registers...\n");
>> break;
>> default:
>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>> index 58f325019886..bec660357eec 100644
>> --- a/drivers/cxl/pci.c
>> +++ b/drivers/cxl/pci.c
>> @@ -796,6 +796,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> struct cxl_register_map map;
>> struct cxl_memdev *cxlmd;
>> int i, rc, pmu_count;
>> + u32 expected, found;
>> bool irq_avail;
>> u16 dvsec;
>>
>> @@ -852,6 +853,17 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> if (rc)
>> dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
>>
>> + /* These are the mandatory capabilities for a Type3 device */
>> + expected = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
>> + BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV);
>> +
>> + if (!cxl_pci_check_caps(cxlds, expected, &found)) {
>> + dev_err(&pdev->dev,
>> + "Expected capabilities not matching with found capabilities: (%08x - %08x)\n",
>> + expected, found);
>> + return -ENXIO;
>> + }
>> +
> Same as above, the capabilities already are cached in cxlds->capabilities. seems like that the 'found' can be removed and using cxlds->capabilities directly here.
>
>
>> rc = cxl_await_media_ready(cxlds);
>> if (rc == 0)
>> cxlds->media_ready = true;
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index 930b1b9c1d6a..4a57bf60403d 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -48,4 +48,6 @@ void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
>> void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
>> int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> enum cxl_resource);
>> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> + u32 *current_caps);
>> #endif
>
>
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 03/20] cxl/pci: add check for validating capabilities
2024-09-10 6:24 ` Li, Ming4
@ 2024-09-10 7:31 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-10 7:31 UTC (permalink / raw)
To: Li, Ming4, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/10/24 07:24, Li, Ming4 wrote:
> On 9/10/2024 11:26 AM, Li, Ming4 wrote:
>> On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
>>> From: Alejandro Lucero <alucerop@amd.com>
>>>
>>> During CXL device initialization supported capabilities by the device
>>> are discovered. Type3 and Type2 devices have different mandatory
>>> capabilities and a Type2 expects a specific set including optional
>>> capabilities.
>>>
>>> Add a function for checking expected capabilities against those found
>>> during initialization.
>>>
>>> Rely on this function for validating capabilities instead of when CXL
>>> regs are probed.
>>>
>>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>>> ---
>>> drivers/cxl/core/pci.c | 17 +++++++++++++++++
>>> drivers/cxl/core/regs.c | 9 ---------
>>> drivers/cxl/pci.c | 12 ++++++++++++
>>> include/linux/cxl/cxl.h | 2 ++
>>> 4 files changed, 31 insertions(+), 9 deletions(-)
>>>
>>> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
>>> index 3d6564dbda57..57370d9beb32 100644
>>> --- a/drivers/cxl/core/pci.c
>>> +++ b/drivers/cxl/core/pci.c
>>> @@ -7,6 +7,7 @@
>>> #include <linux/pci.h>
>>> #include <linux/pci-doe.h>
>>> #include <linux/aer.h>
>>> +#include <linux/cxl/cxl.h>
>>> #include <linux/cxl/pci.h>
>>> #include <cxlpci.h>
>>> #include <cxlmem.h>
>>> @@ -1077,3 +1078,19 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
>>> __cxl_endpoint_decoder_reset_detected);
>>> }
>>> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
>>> +
>>> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>>> + u32 *current_caps)
>>> +{
>>> + if (current_caps)
>>> + *current_caps = cxlds->capabilities;
>>> +
>>> + dev_dbg(cxlds->dev, "Checking cxlds caps 0x%08x vs expected caps 0x%08x\n",
>>> + cxlds->capabilities, expected_caps);
>>> +
>>> + if ((cxlds->capabilities & expected_caps) != expected_caps)
>>> + return false;
>>> +
>>> + return true;
>>> +}
>>> +EXPORT_SYMBOL_NS_GPL(cxl_pci_check_caps, CXL);
>> Why has to use this 'u32 *current_caps' as a parameter? if user wants to know the capabilities of a device, they can get it from cxlds->capabilities directly.
>>
> Sorry, I missed something implemented in PATCH #1, seems like you can not access struct cxl_dev_state from efx driver side. right?
Yes. The idea is to avoid that for facilitating changes to those updated
structs. Initially, during the RFC review, it was said let's enforce
because accel drivers should not be trusted, but I think the main
concern is maintenance.
>
>>> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
>>> index 8b8abcadcb93..35f6dc97be6e 100644
>>> --- a/drivers/cxl/core/regs.c
>>> +++ b/drivers/cxl/core/regs.c
>>> @@ -443,15 +443,6 @@ static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
>>> case CXL_REGLOC_RBI_MEMDEV:
>>> dev_map = &map->device_map;
>>> cxl_probe_device_regs(host, base, dev_map, caps);
>>> - if (!dev_map->status.valid || !dev_map->mbox.valid ||
>>> - !dev_map->memdev.valid) {
>>> - dev_err(host, "registers not found: %s%s%s\n",
>>> - !dev_map->status.valid ? "status " : "",
>>> - !dev_map->mbox.valid ? "mbox " : "",
>>> - !dev_map->memdev.valid ? "memdev " : "");
>>> - return -ENXIO;
>>> - }
>>> -
>>> dev_dbg(host, "Probing device registers...\n");
>>> break;
>>> default:
>>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>>> index 58f325019886..bec660357eec 100644
>>> --- a/drivers/cxl/pci.c
>>> +++ b/drivers/cxl/pci.c
>>> @@ -796,6 +796,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>>> struct cxl_register_map map;
>>> struct cxl_memdev *cxlmd;
>>> int i, rc, pmu_count;
>>> + u32 expected, found;
>>> bool irq_avail;
>>> u16 dvsec;
>>>
>>> @@ -852,6 +853,17 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>>> if (rc)
>>> dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
>>>
>>> + /* These are the mandatory capabilities for a Type3 device */
>>> + expected = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
>>> + BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV);
>>> +
>>> + if (!cxl_pci_check_caps(cxlds, expected, &found)) {
>>> + dev_err(&pdev->dev,
>>> + "Expected capabilities not matching with found capabilities: (%08x - %08x)\n",
>>> + expected, found);
>>> + return -ENXIO;
>>> + }
>>> +
>> Same as above, the capabilities already are cached in cxlds->capabilities. seems like that the 'found' can be removed and using cxlds->capabilities directly here.
>>
>>
>>> rc = cxl_await_media_ready(cxlds);
>>> if (rc == 0)
>>> cxlds->media_ready = true;
>>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>>> index 930b1b9c1d6a..4a57bf60403d 100644
>>> --- a/include/linux/cxl/cxl.h
>>> +++ b/include/linux/cxl/cxl.h
>>> @@ -48,4 +48,6 @@ void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
>>> void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
>>> int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>>> enum cxl_resource);
>>> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>>> + u32 *current_caps);
>>> #endif
>>
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 03/20] cxl/pci: add check for validating capabilities
2024-09-07 8:18 ` [PATCH v3 03/20] cxl/pci: add check for validating capabilities alejandro.lucero-palau
2024-09-10 3:26 ` Li, Ming4
@ 2024-09-11 23:06 ` Dave Jiang
2024-09-16 8:56 ` Alejandro Lucero Palau
2024-09-13 17:28 ` Jonathan Cameron
2 siblings, 1 reply; 88+ messages in thread
From: Dave Jiang @ 2024-09-11 23:06 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> During CXL device initialization supported capabilities by the device
> are discovered. Type3 and Type2 devices have different mandatory
> capabilities and a Type2 expects a specific set including optional
> capabilities.
>
> Add a function for checking expected capabilities against those found
> during initialization.
>
> Rely on this function for validating capabilities instead of when CXL
> regs are probed.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/pci.c | 17 +++++++++++++++++
> drivers/cxl/core/regs.c | 9 ---------
> drivers/cxl/pci.c | 12 ++++++++++++
> include/linux/cxl/cxl.h | 2 ++
> 4 files changed, 31 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> index 3d6564dbda57..57370d9beb32 100644
> --- a/drivers/cxl/core/pci.c
> +++ b/drivers/cxl/core/pci.c
> @@ -7,6 +7,7 @@
> #include <linux/pci.h>
> #include <linux/pci-doe.h>
> #include <linux/aer.h>
> +#include <linux/cxl/cxl.h>
> #include <linux/cxl/pci.h>
> #include <cxlpci.h>
> #include <cxlmem.h>
> @@ -1077,3 +1078,19 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
> __cxl_endpoint_decoder_reset_detected);
> }
> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
> +
> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> + u32 *current_caps)
> +{
> + if (current_caps)
> + *current_caps = cxlds->capabilities;
> +
> + dev_dbg(cxlds->dev, "Checking cxlds caps 0x%08x vs expected caps 0x%08x\n",
> + cxlds->capabilities, expected_caps);
> +
> + if ((cxlds->capabilities & expected_caps) != expected_caps)
> + return false;
> +
> + return true;
I think you can just do
return (cxlds->capabilities & expected_caps) == expected_caps;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_pci_check_caps, CXL);
> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
> index 8b8abcadcb93..35f6dc97be6e 100644
> --- a/drivers/cxl/core/regs.c
> +++ b/drivers/cxl/core/regs.c
> @@ -443,15 +443,6 @@ static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
> case CXL_REGLOC_RBI_MEMDEV:
> dev_map = &map->device_map;
> cxl_probe_device_regs(host, base, dev_map, caps);
> - if (!dev_map->status.valid || !dev_map->mbox.valid ||
> - !dev_map->memdev.valid) {
> - dev_err(host, "registers not found: %s%s%s\n",
> - !dev_map->status.valid ? "status " : "",
> - !dev_map->mbox.valid ? "mbox " : "",
> - !dev_map->memdev.valid ? "memdev " : "");
> - return -ENXIO;
> - }
> -
> dev_dbg(host, "Probing device registers...\n");
> break;
> default:
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 58f325019886..bec660357eec 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -796,6 +796,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> struct cxl_register_map map;
> struct cxl_memdev *cxlmd;
> int i, rc, pmu_count;
> + u32 expected, found;
> bool irq_avail;
> u16 dvsec;
>
> @@ -852,6 +853,17 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> if (rc)
> dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
>
> + /* These are the mandatory capabilities for a Type3 device */
> + expected = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
> + BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV);
Maybe we can create a static mask for the expected mandatory type3 caps.
I also wonder if cxl_pci_check_caps() can key on pci device type or class type and know which expected mask to use and no need to pass in the expected mask. Or since the driver is being attached to a certain device type, it should know what it is already, maybe we can attach static driver data to it that can be retrieved as the expected_caps:
struct cxl_driver_data {
u32 expected_caps;
};
static struct cxl_driver_data cxl_driver_data = {
.expected_caps = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV),
};
static const struct pci_device_id cxl_mem_pci_tbl[] = {
/* Maybe need a new PCI_DEVICE_CLASS_DATA() macro */
{
.class = (PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF),
.class_mask = ~0,
.vendor = PCI_ANY_ID,
.device = PCI_ANY_ID,
.sub_vendor = PCI_ANY_ID,
.subdevice = PCI_ANY_ID,
.driver_data = (kernel_ulong_t)&cxl_driver_data,
},
{},
};
MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl);
static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct cxl_driver_data *data = (struct cxl_driver_data *)id->driver_data;
rc = cxl_pci_check_caps(cxlds, data->expected_caps, &found);
....
}
> +
> + if (!cxl_pci_check_caps(cxlds, expected, &found)) {
> + dev_err(&pdev->dev,
> + "Expected capabilities not matching with found capabilities: (%08x - %08x)\n",
> + expected, found);
> + return -ENXIO;
> + }
> +
> rc = cxl_await_media_ready(cxlds);
> if (rc == 0)
> cxlds->media_ready = true;
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index 930b1b9c1d6a..4a57bf60403d 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -48,4 +48,6 @@ void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
> void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
> int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> enum cxl_resource);
> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> + u32 *current_caps);
> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 03/20] cxl/pci: add check for validating capabilities
2024-09-11 23:06 ` Dave Jiang
@ 2024-09-16 8:56 ` Alejandro Lucero Palau
2024-09-16 16:11 ` Dave Jiang
0 siblings, 1 reply; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 8:56 UTC (permalink / raw)
To: Dave Jiang, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/12/24 00:06, Dave Jiang wrote:
>
> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> During CXL device initialization supported capabilities by the device
>> are discovered. Type3 and Type2 devices have different mandatory
>> capabilities and a Type2 expects a specific set including optional
>> capabilities.
>>
>> Add a function for checking expected capabilities against those found
>> during initialization.
>>
>> Rely on this function for validating capabilities instead of when CXL
>> regs are probed.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/pci.c | 17 +++++++++++++++++
>> drivers/cxl/core/regs.c | 9 ---------
>> drivers/cxl/pci.c | 12 ++++++++++++
>> include/linux/cxl/cxl.h | 2 ++
>> 4 files changed, 31 insertions(+), 9 deletions(-)
>>
>> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
>> index 3d6564dbda57..57370d9beb32 100644
>> --- a/drivers/cxl/core/pci.c
>> +++ b/drivers/cxl/core/pci.c
>> @@ -7,6 +7,7 @@
>> #include <linux/pci.h>
>> #include <linux/pci-doe.h>
>> #include <linux/aer.h>
>> +#include <linux/cxl/cxl.h>
>> #include <linux/cxl/pci.h>
>> #include <cxlpci.h>
>> #include <cxlmem.h>
>> @@ -1077,3 +1078,19 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
>> __cxl_endpoint_decoder_reset_detected);
>> }
>> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
>> +
>> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> + u32 *current_caps)
>> +{
>> + if (current_caps)
>> + *current_caps = cxlds->capabilities;
>> +
>> + dev_dbg(cxlds->dev, "Checking cxlds caps 0x%08x vs expected caps 0x%08x\n",
>> + cxlds->capabilities, expected_caps);
>> +
>> + if ((cxlds->capabilities & expected_caps) != expected_caps)
>> + return false;
>> +
>> + return true;
>
> I think you can just do
> return (cxlds->capabilities & expected_caps) == expected_caps;
Yes. I'll do.
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_pci_check_caps, CXL);
>> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
>> index 8b8abcadcb93..35f6dc97be6e 100644
>> --- a/drivers/cxl/core/regs.c
>> +++ b/drivers/cxl/core/regs.c
>> @@ -443,15 +443,6 @@ static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
>> case CXL_REGLOC_RBI_MEMDEV:
>> dev_map = &map->device_map;
>> cxl_probe_device_regs(host, base, dev_map, caps);
>> - if (!dev_map->status.valid || !dev_map->mbox.valid ||
>> - !dev_map->memdev.valid) {
>> - dev_err(host, "registers not found: %s%s%s\n",
>> - !dev_map->status.valid ? "status " : "",
>> - !dev_map->mbox.valid ? "mbox " : "",
>> - !dev_map->memdev.valid ? "memdev " : "");
>> - return -ENXIO;
>> - }
>> -
>> dev_dbg(host, "Probing device registers...\n");
>> break;
>> default:
>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>> index 58f325019886..bec660357eec 100644
>> --- a/drivers/cxl/pci.c
>> +++ b/drivers/cxl/pci.c
>> @@ -796,6 +796,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> struct cxl_register_map map;
>> struct cxl_memdev *cxlmd;
>> int i, rc, pmu_count;
>> + u32 expected, found;
>> bool irq_avail;
>> u16 dvsec;
>>
>> @@ -852,6 +853,17 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> if (rc)
>> dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
>>
>> + /* These are the mandatory capabilities for a Type3 device */
>> + expected = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
>> + BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV);
> Maybe we can create a static mask for the expected mandatory type3 caps.
>
> I also wonder if cxl_pci_check_caps() can key on pci device type or class type and know which expected mask to use and no need to pass in the expected mask. Or since the driver is being attached to a certain device type, it should know what it is already, maybe we can attach static driver data to it that can be retrieved as the expected_caps:
>
> struct cxl_driver_data {
> u32 expected_caps;
> };
>
> static struct cxl_driver_data cxl_driver_data = {
> .expected_caps = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
> BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV),
> };
>
> static const struct pci_device_id cxl_mem_pci_tbl[] = {
> /* Maybe need a new PCI_DEVICE_CLASS_DATA() macro */
> {
> .class = (PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF),
> .class_mask = ~0,
> .vendor = PCI_ANY_ID,
> .device = PCI_ANY_ID,
> .sub_vendor = PCI_ANY_ID,
> .subdevice = PCI_ANY_ID,
> .driver_data = (kernel_ulong_t)&cxl_driver_data,
> },
> {},
> };
> MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl);
>
> static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> {
> struct cxl_driver_data *data = (struct cxl_driver_data *)id->driver_data;
>
> rc = cxl_pci_check_caps(cxlds, data->expected_caps, &found);
> ....
> }
I do not think this requires so much plumbing, but maybe using a macro
for type3 mandatory caps makes sense.
About accel drivers, I'm not sure a macro is suitable because, I think,
when hopefully CXL designs become mainstream, the caps will likely be
needed to be extracted from the device somehow, other than what the CXL
regs tell themselves, just for the sake of double checking. Thinking
here of same base device with some changes regarding CXL in different
device models/flavors.
>> +
>> + if (!cxl_pci_check_caps(cxlds, expected, &found)) {
>> + dev_err(&pdev->dev,
>> + "Expected capabilities not matching with found capabilities: (%08x - %08x)\n",
>> + expected, found);
>> + return -ENXIO;
>> + }
>> +
>> rc = cxl_await_media_ready(cxlds);
>> if (rc == 0)
>> cxlds->media_ready = true;
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index 930b1b9c1d6a..4a57bf60403d 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -48,4 +48,6 @@ void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
>> void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
>> int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> enum cxl_resource);
>> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> + u32 *current_caps);
>> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 03/20] cxl/pci: add check for validating capabilities
2024-09-16 8:56 ` Alejandro Lucero Palau
@ 2024-09-16 16:11 ` Dave Jiang
0 siblings, 0 replies; 88+ messages in thread
From: Dave Jiang @ 2024-09-16 16:11 UTC (permalink / raw)
To: Alejandro Lucero Palau, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/16/24 1:56 AM, Alejandro Lucero Palau wrote:
>
> On 9/12/24 00:06, Dave Jiang wrote:
>>
>> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>>> From: Alejandro Lucero <alucerop@amd.com>
>>>
>>> During CXL device initialization supported capabilities by the device
>>> are discovered. Type3 and Type2 devices have different mandatory
>>> capabilities and a Type2 expects a specific set including optional
>>> capabilities.
>>>
>>> Add a function for checking expected capabilities against those found
>>> during initialization.
>>>
>>> Rely on this function for validating capabilities instead of when CXL
>>> regs are probed.
>>>
>>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>>> ---
>>> drivers/cxl/core/pci.c | 17 +++++++++++++++++
>>> drivers/cxl/core/regs.c | 9 ---------
>>> drivers/cxl/pci.c | 12 ++++++++++++
>>> include/linux/cxl/cxl.h | 2 ++
>>> 4 files changed, 31 insertions(+), 9 deletions(-)
>>>
>>> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
>>> index 3d6564dbda57..57370d9beb32 100644
>>> --- a/drivers/cxl/core/pci.c
>>> +++ b/drivers/cxl/core/pci.c
>>> @@ -7,6 +7,7 @@
>>> #include <linux/pci.h>
>>> #include <linux/pci-doe.h>
>>> #include <linux/aer.h>
>>> +#include <linux/cxl/cxl.h>
>>> #include <linux/cxl/pci.h>
>>> #include <cxlpci.h>
>>> #include <cxlmem.h>
>>> @@ -1077,3 +1078,19 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
>>> __cxl_endpoint_decoder_reset_detected);
>>> }
>>> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
>>> +
>>> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>>> + u32 *current_caps)
>>> +{
>>> + if (current_caps)
>>> + *current_caps = cxlds->capabilities;
>>> +
>>> + dev_dbg(cxlds->dev, "Checking cxlds caps 0x%08x vs expected caps 0x%08x\n",
>>> + cxlds->capabilities, expected_caps);
>>> +
>>> + if ((cxlds->capabilities & expected_caps) != expected_caps)
>>> + return false;
>>> +
>>> + return true;
>>
>> I think you can just do
>> return (cxlds->capabilities & expected_caps) == expected_caps;
>
>
> Yes. I'll do.
>
>
>>> +}
>>> +EXPORT_SYMBOL_NS_GPL(cxl_pci_check_caps, CXL);
>>> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
>>> index 8b8abcadcb93..35f6dc97be6e 100644
>>> --- a/drivers/cxl/core/regs.c
>>> +++ b/drivers/cxl/core/regs.c
>>> @@ -443,15 +443,6 @@ static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
>>> case CXL_REGLOC_RBI_MEMDEV:
>>> dev_map = &map->device_map;
>>> cxl_probe_device_regs(host, base, dev_map, caps);
>>> - if (!dev_map->status.valid || !dev_map->mbox.valid ||
>>> - !dev_map->memdev.valid) {
>>> - dev_err(host, "registers not found: %s%s%s\n",
>>> - !dev_map->status.valid ? "status " : "",
>>> - !dev_map->mbox.valid ? "mbox " : "",
>>> - !dev_map->memdev.valid ? "memdev " : "");
>>> - return -ENXIO;
>>> - }
>>> -
>>> dev_dbg(host, "Probing device registers...\n");
>>> break;
>>> default:
>>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>>> index 58f325019886..bec660357eec 100644
>>> --- a/drivers/cxl/pci.c
>>> +++ b/drivers/cxl/pci.c
>>> @@ -796,6 +796,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>>> struct cxl_register_map map;
>>> struct cxl_memdev *cxlmd;
>>> int i, rc, pmu_count;
>>> + u32 expected, found;
>>> bool irq_avail;
>>> u16 dvsec;
>>> @@ -852,6 +853,17 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>>> if (rc)
>>> dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
>>> + /* These are the mandatory capabilities for a Type3 device */
>>> + expected = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
>>> + BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV);
>> Maybe we can create a static mask for the expected mandatory type3 caps.
>>
>> I also wonder if cxl_pci_check_caps() can key on pci device type or class type and know which expected mask to use and no need to pass in the expected mask. Or since the driver is being attached to a certain device type, it should know what it is already, maybe we can attach static driver data to it that can be retrieved as the expected_caps:
>>
>> struct cxl_driver_data {
>> u32 expected_caps;
>> };
>>
>> static struct cxl_driver_data cxl_driver_data = {
>> .expected_caps = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
>> BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV),
>> };
>>
>> static const struct pci_device_id cxl_mem_pci_tbl[] = {
>> /* Maybe need a new PCI_DEVICE_CLASS_DATA() macro */
>> {
>> .class = (PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF),
>> .class_mask = ~0,
>> .vendor = PCI_ANY_ID,
>> .device = PCI_ANY_ID,
>> .sub_vendor = PCI_ANY_ID,
>> .subdevice = PCI_ANY_ID,
>> .driver_data = (kernel_ulong_t)&cxl_driver_data,
>> },
>> {},
>> };
>> MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl);
>>
>> static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> {
>> struct cxl_driver_data *data = (struct cxl_driver_data *)id->driver_data;
>>
>> rc = cxl_pci_check_caps(cxlds, data->expected_caps, &found);
>> ....
>> }
>
>
> I do not think this requires so much plumbing, but maybe using a macro for type3 mandatory caps makes sense.
ok sure.
>
> About accel drivers, I'm not sure a macro is suitable because, I think, when hopefully CXL designs become mainstream, the caps will likely be needed to be extracted from the device somehow, other than what the CXL regs tell themselves, just for the sake of double checking. Thinking here of same base device with some changes regarding CXL in different device models/flavors.
I can see type2 have very different mandatory bits depending on the device. Maybe there's a small set of must haves for all.
>
>
>>> +
>>> + if (!cxl_pci_check_caps(cxlds, expected, &found)) {
>>> + dev_err(&pdev->dev,
>>> + "Expected capabilities not matching with found capabilities: (%08x - %08x)\n",
>>> + expected, found);
>>> + return -ENXIO;
>>> + }
>>> +
>>> rc = cxl_await_media_ready(cxlds);
>>> if (rc == 0)
>>> cxlds->media_ready = true;
>>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>>> index 930b1b9c1d6a..4a57bf60403d 100644
>>> --- a/include/linux/cxl/cxl.h
>>> +++ b/include/linux/cxl/cxl.h
>>> @@ -48,4 +48,6 @@ void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
>>> void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
>>> int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>>> enum cxl_resource);
>>> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>>> + u32 *current_caps);
>>> #endif
>
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 03/20] cxl/pci: add check for validating capabilities
2024-09-07 8:18 ` [PATCH v3 03/20] cxl/pci: add check for validating capabilities alejandro.lucero-palau
2024-09-10 3:26 ` Li, Ming4
2024-09-11 23:06 ` Dave Jiang
@ 2024-09-13 17:28 ` Jonathan Cameron
2024-09-16 12:17 ` Alejandro Lucero Palau
2 siblings, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 17:28 UTC (permalink / raw)
To: alejandro.lucero-palau, netdev
Cc: linux-cxl, dan.j.williams, martin.habets, edward.cree, davem,
kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:19 +0100
<alejandro.lucero-palau@amd.com> wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> During CXL device initialization supported capabilities by the device
> are discovered. Type3 and Type2 devices have different mandatory
> capabilities and a Type2 expects a specific set including optional
> capabilities.
>
> Add a function for checking expected capabilities against those found
> during initialization.
>
> Rely on this function for validating capabilities instead of when CXL
> regs are probed.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/pci.c | 17 +++++++++++++++++
> drivers/cxl/core/regs.c | 9 ---------
> drivers/cxl/pci.c | 12 ++++++++++++
> include/linux/cxl/cxl.h | 2 ++
> 4 files changed, 31 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> index 3d6564dbda57..57370d9beb32 100644
> --- a/drivers/cxl/core/pci.c
> +++ b/drivers/cxl/core/pci.c
> @@ -7,6 +7,7 @@
> #include <linux/pci.h>
> #include <linux/pci-doe.h>
> #include <linux/aer.h>
> +#include <linux/cxl/cxl.h>
> #include <linux/cxl/pci.h>
> #include <cxlpci.h>
> #include <cxlmem.h>
> @@ -1077,3 +1078,19 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
> __cxl_endpoint_decoder_reset_detected);
> }
> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
> +
> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> + u32 *current_caps)
> +{
> + if (current_caps)
> + *current_caps = cxlds->capabilities;
I'd split this up as setting a value in a 'check_caps' and comparisom with
a list is odd.
Also bitmaps all the way would be better.
Given you know it fits in one unsigned long you can short cut the
assignment of the bits though. Easy to extend that later if the bitmap
gets bigger.
> +
> + dev_dbg(cxlds->dev, "Checking cxlds caps 0x%08x vs expected caps 0x%08x\n",
> + cxlds->capabilities, expected_caps);
> +
> + if ((cxlds->capabilities & expected_caps) != expected_caps)
> + return false;
> +
> + return true;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_pci_check_caps, CXL);
> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
> index 8b8abcadcb93..35f6dc97be6e 100644
> --- a/drivers/cxl/core/regs.c
> +++ b/drivers/cxl/core/regs.c
> @@ -443,15 +443,6 @@ static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
> case CXL_REGLOC_RBI_MEMDEV:
> dev_map = &map->device_map;
> cxl_probe_device_regs(host, base, dev_map, caps);
> - if (!dev_map->status.valid || !dev_map->mbox.valid ||
> - !dev_map->memdev.valid) {
> - dev_err(host, "registers not found: %s%s%s\n",
> - !dev_map->status.valid ? "status " : "",
> - !dev_map->mbox.valid ? "mbox " : "",
> - !dev_map->memdev.valid ? "memdev " : "");
> - return -ENXIO;
> - }
> -
> dev_dbg(host, "Probing device registers...\n");
> break;
> default:
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 58f325019886..bec660357eec 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -796,6 +796,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> struct cxl_register_map map;
> struct cxl_memdev *cxlmd;
> int i, rc, pmu_count;
> + u32 expected, found;
> bool irq_avail;
> u16 dvsec;
>
> @@ -852,6 +853,17 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> if (rc)
> dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
>
> + /* These are the mandatory capabilities for a Type3 device */
> + expected = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
> + BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV);
> +
> + if (!cxl_pci_check_caps(cxlds, expected, &found)) {
> + dev_err(&pdev->dev,
> + "Expected capabilities not matching with found capabilities: (%08x - %08x)\n",
> + expected, found);
> + return -ENXIO;
> + }
> +
> rc = cxl_await_media_ready(cxlds);
> if (rc == 0)
> cxlds->media_ready = true;
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index 930b1b9c1d6a..4a57bf60403d 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -48,4 +48,6 @@ void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
> void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
> int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> enum cxl_resource);
> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> + u32 *current_caps);
> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 03/20] cxl/pci: add check for validating capabilities
2024-09-13 17:28 ` Jonathan Cameron
@ 2024-09-16 12:17 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 12:17 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau, netdev
Cc: linux-cxl, dan.j.williams, martin.habets, edward.cree, davem,
kuba, pabeni, edumazet
On 9/13/24 18:28, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:19 +0100
> <alejandro.lucero-palau@amd.com> wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> During CXL device initialization supported capabilities by the device
>> are discovered. Type3 and Type2 devices have different mandatory
>> capabilities and a Type2 expects a specific set including optional
>> capabilities.
>>
>> Add a function for checking expected capabilities against those found
>> during initialization.
>>
>> Rely on this function for validating capabilities instead of when CXL
>> regs are probed.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/pci.c | 17 +++++++++++++++++
>> drivers/cxl/core/regs.c | 9 ---------
>> drivers/cxl/pci.c | 12 ++++++++++++
>> include/linux/cxl/cxl.h | 2 ++
>> 4 files changed, 31 insertions(+), 9 deletions(-)
>>
>> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
>> index 3d6564dbda57..57370d9beb32 100644
>> --- a/drivers/cxl/core/pci.c
>> +++ b/drivers/cxl/core/pci.c
>> @@ -7,6 +7,7 @@
>> #include <linux/pci.h>
>> #include <linux/pci-doe.h>
>> #include <linux/aer.h>
>> +#include <linux/cxl/cxl.h>
>> #include <linux/cxl/pci.h>
>> #include <cxlpci.h>
>> #include <cxlmem.h>
>> @@ -1077,3 +1078,19 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
>> __cxl_endpoint_decoder_reset_detected);
>> }
>> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
>> +
>> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> + u32 *current_caps)
>> +{
>> + if (current_caps)
>> + *current_caps = cxlds->capabilities;
> I'd split this up as setting a value in a 'check_caps' and comparisom with
> a list is odd.
The idea is to return the caps discovered for the accel driver to know
which one is not there.
The other option is to print out here a message which does not seem
right to me.
> Also bitmaps all the way would be better.
> Given you know it fits in one unsigned long you can short cut the
> assignment of the bits though. Easy to extend that later if the bitmap
> gets bigger.
Yes, I'll adapt it to using a bitmap.
Thanks
>
>> +
>> + dev_dbg(cxlds->dev, "Checking cxlds caps 0x%08x vs expected caps 0x%08x\n",
>> + cxlds->capabilities, expected_caps);
>> +
>> + if ((cxlds->capabilities & expected_caps) != expected_caps)
>> + return false;
>> +
>> + return true;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_pci_check_caps, CXL);
>> diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
>> index 8b8abcadcb93..35f6dc97be6e 100644
>> --- a/drivers/cxl/core/regs.c
>> +++ b/drivers/cxl/core/regs.c
>> @@ -443,15 +443,6 @@ static int cxl_probe_regs(struct cxl_register_map *map, u32 *caps)
>> case CXL_REGLOC_RBI_MEMDEV:
>> dev_map = &map->device_map;
>> cxl_probe_device_regs(host, base, dev_map, caps);
>> - if (!dev_map->status.valid || !dev_map->mbox.valid ||
>> - !dev_map->memdev.valid) {
>> - dev_err(host, "registers not found: %s%s%s\n",
>> - !dev_map->status.valid ? "status " : "",
>> - !dev_map->mbox.valid ? "mbox " : "",
>> - !dev_map->memdev.valid ? "memdev " : "");
>> - return -ENXIO;
>> - }
>> -
>> dev_dbg(host, "Probing device registers...\n");
>> break;
>> default:
>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>> index 58f325019886..bec660357eec 100644
>> --- a/drivers/cxl/pci.c
>> +++ b/drivers/cxl/pci.c
>> @@ -796,6 +796,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> struct cxl_register_map map;
>> struct cxl_memdev *cxlmd;
>> int i, rc, pmu_count;
>> + u32 expected, found;
>> bool irq_avail;
>> u16 dvsec;
>>
>> @@ -852,6 +853,17 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>> if (rc)
>> dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
>>
>> + /* These are the mandatory capabilities for a Type3 device */
>> + expected = BIT(CXL_DEV_CAP_HDM) | BIT(CXL_DEV_CAP_DEV_STATUS) |
>> + BIT(CXL_DEV_CAP_MAILBOX_PRIMARY) | BIT(CXL_DEV_CAP_MEMDEV);
>> +
>> + if (!cxl_pci_check_caps(cxlds, expected, &found)) {
>> + dev_err(&pdev->dev,
>> + "Expected capabilities not matching with found capabilities: (%08x - %08x)\n",
>> + expected, found);
>> + return -ENXIO;
>> + }
>> +
>> rc = cxl_await_media_ready(cxlds);
>> if (rc == 0)
>> cxlds->media_ready = true;
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index 930b1b9c1d6a..4a57bf60403d 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -48,4 +48,6 @@ void cxl_set_dvsec(struct cxl_dev_state *cxlds, u16 dvsec);
>> void cxl_set_serial(struct cxl_dev_state *cxlds, u64 serial);
>> int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> enum cxl_resource);
>> +bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> + u32 *current_caps);
>> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 04/20] cxl: move pci generic code
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (2 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 03/20] cxl/pci: add check for validating capabilities alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-11 23:55 ` Dave Jiang
2024-09-07 8:18 ` [PATCH v3 05/20] cxl: add function for type2 cxl regs setup alejandro.lucero-palau
` (15 subsequent siblings)
19 siblings, 1 reply; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Inside cxl/core/pci.c there are helpers for CXL PCIe initialization
meanwhile cxl/pci.c implements the functionality for a Type3 device
initialization.
Move those functions required also for Type2 initialization to
cxl/core/pci.c with a specific function using that moved code added in
a following patch.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/pci.c | 63 ++++++++++++++++++++++++++++++++++++++++++
drivers/cxl/cxlpci.h | 3 ++
drivers/cxl/pci.c | 60 ----------------------------------------
3 files changed, 66 insertions(+), 60 deletions(-)
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index 57370d9beb32..bf57f081ef8f 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -1079,6 +1079,69 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
}
EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
+/*
+ * Assume that any RCIEP that emits the CXL memory expander class code
+ * is an RCD
+ */
+bool is_cxl_restricted(struct pci_dev *pdev)
+{
+ return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END;
+}
+EXPORT_SYMBOL_NS_GPL(is_cxl_restricted, CXL);
+
+static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
+ struct cxl_register_map *map)
+{
+ struct cxl_port *port;
+ struct cxl_dport *dport;
+ resource_size_t component_reg_phys;
+
+ *map = (struct cxl_register_map) {
+ .host = &pdev->dev,
+ .resource = CXL_RESOURCE_NONE,
+ };
+
+ port = cxl_pci_find_port(pdev, &dport);
+ if (!port)
+ return -EPROBE_DEFER;
+
+ component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport);
+
+ put_device(&port->dev);
+
+ if (component_reg_phys == CXL_RESOURCE_NONE)
+ return -ENXIO;
+
+ map->resource = component_reg_phys;
+ map->reg_type = CXL_REGLOC_RBI_COMPONENT;
+ map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
+
+ return 0;
+}
+
+int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
+ struct cxl_register_map *map,
+ u32 *caps)
+{
+ int rc;
+
+ rc = cxl_find_regblock(pdev, type, map);
+
+ /*
+ * If the Register Locator DVSEC does not exist, check if it
+ * is an RCH and try to extract the Component Registers from
+ * an RCRB.
+ */
+ if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev))
+ rc = cxl_rcrb_get_comp_regs(pdev, map);
+
+ if (rc)
+ return rc;
+
+ return cxl_setup_regs(map, caps);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, CXL);
+
bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
u32 *current_caps)
{
diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
index eb59019fe5f3..786b811effba 100644
--- a/drivers/cxl/cxlpci.h
+++ b/drivers/cxl/cxlpci.h
@@ -113,4 +113,7 @@ void read_cdat_data(struct cxl_port *port);
void cxl_cor_error_detected(struct pci_dev *pdev);
pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
pci_channel_state_t state);
+bool is_cxl_restricted(struct pci_dev *pdev);
+int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
+ struct cxl_register_map *map, u32 *caps);
#endif /* __CXL_PCI_H__ */
diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
index bec660357eec..2b85f87549c2 100644
--- a/drivers/cxl/pci.c
+++ b/drivers/cxl/pci.c
@@ -463,66 +463,6 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
return 0;
}
-/*
- * Assume that any RCIEP that emits the CXL memory expander class code
- * is an RCD
- */
-static bool is_cxl_restricted(struct pci_dev *pdev)
-{
- return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END;
-}
-
-static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
- struct cxl_register_map *map)
-{
- struct cxl_port *port;
- struct cxl_dport *dport;
- resource_size_t component_reg_phys;
-
- *map = (struct cxl_register_map) {
- .host = &pdev->dev,
- .resource = CXL_RESOURCE_NONE,
- };
-
- port = cxl_pci_find_port(pdev, &dport);
- if (!port)
- return -EPROBE_DEFER;
-
- component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport);
-
- put_device(&port->dev);
-
- if (component_reg_phys == CXL_RESOURCE_NONE)
- return -ENXIO;
-
- map->resource = component_reg_phys;
- map->reg_type = CXL_REGLOC_RBI_COMPONENT;
- map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
-
- return 0;
-}
-
-static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
- struct cxl_register_map *map, u32 *caps)
-{
- int rc;
-
- rc = cxl_find_regblock(pdev, type, map);
-
- /*
- * If the Register Locator DVSEC does not exist, check if it
- * is an RCH and try to extract the Component Registers from
- * an RCRB.
- */
- if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev))
- rc = cxl_rcrb_get_comp_regs(pdev, map);
-
- if (rc)
- return rc;
-
- return cxl_setup_regs(map, caps);
-}
-
static int cxl_pci_ras_unmask(struct pci_dev *pdev)
{
struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 04/20] cxl: move pci generic code
2024-09-07 8:18 ` [PATCH v3 04/20] cxl: move pci generic code alejandro.lucero-palau
@ 2024-09-11 23:55 ` Dave Jiang
2024-09-16 9:46 ` Alejandro Lucero Palau
0 siblings, 1 reply; 88+ messages in thread
From: Dave Jiang @ 2024-09-11 23:55 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Inside cxl/core/pci.c there are helpers for CXL PCIe initialization
> meanwhile cxl/pci.c implements the functionality for a Type3 device
> initialization.
>
> Move those functions required also for Type2 initialization to
> cxl/core/pci.c with a specific function using that moved code added in
> a following patch.
Please consider rephrasing as:
Move helper functions from cxl/pci.c to cxl/core/pci.c in order to be
exported and shared with CXL Type2 device initialization.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/pci.c | 63 ++++++++++++++++++++++++++++++++++++++++++
> drivers/cxl/cxlpci.h | 3 ++
> drivers/cxl/pci.c | 60 ----------------------------------------
> 3 files changed, 66 insertions(+), 60 deletions(-)
>
> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> index 57370d9beb32..bf57f081ef8f 100644
> --- a/drivers/cxl/core/pci.c
> +++ b/drivers/cxl/core/pci.c
> @@ -1079,6 +1079,69 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
> }
> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
>
> +/*
> + * Assume that any RCIEP that emits the CXL memory expander class code
> + * is an RCD
> + */
> +bool is_cxl_restricted(struct pci_dev *pdev)
> +{
> + return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END;
> +}
> +EXPORT_SYMBOL_NS_GPL(is_cxl_restricted, CXL);
> +
> +static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
> + struct cxl_register_map *map)
> +{
> + struct cxl_port *port;
> + struct cxl_dport *dport;
> + resource_size_t component_reg_phys;
> +
> + *map = (struct cxl_register_map) {
> + .host = &pdev->dev,
> + .resource = CXL_RESOURCE_NONE,
> + };
> +
> + port = cxl_pci_find_port(pdev, &dport);
> + if (!port)
> + return -EPROBE_DEFER;
> +
> + component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport);
> +
> + put_device(&port->dev);
> +
> + if (component_reg_phys == CXL_RESOURCE_NONE)
> + return -ENXIO;
> +
> + map->resource = component_reg_phys;
> + map->reg_type = CXL_REGLOC_RBI_COMPONENT;
> + map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
> +
> + return 0;
> +}
> +
> +int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
> + struct cxl_register_map *map,
> + u32 *caps)
> +{
> + int rc;
> +
> + rc = cxl_find_regblock(pdev, type, map);
> +
> + /*
> + * If the Register Locator DVSEC does not exist, check if it
> + * is an RCH and try to extract the Component Registers from
> + * an RCRB.
> + */
> + if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev))
> + rc = cxl_rcrb_get_comp_regs(pdev, map);
> +
> + if (rc)
> + return rc;
> +
> + return cxl_setup_regs(map, caps);
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, CXL);
> +
> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> u32 *current_caps)
> {
> diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
> index eb59019fe5f3..786b811effba 100644
> --- a/drivers/cxl/cxlpci.h
> +++ b/drivers/cxl/cxlpci.h
> @@ -113,4 +113,7 @@ void read_cdat_data(struct cxl_port *port);
> void cxl_cor_error_detected(struct pci_dev *pdev);
> pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
> pci_channel_state_t state);
> +bool is_cxl_restricted(struct pci_dev *pdev);
> +int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
> + struct cxl_register_map *map, u32 *caps);
Does this need to go to a different header like include/cxl/pci.h or something for type2 consumption?
DJ
> #endif /* __CXL_PCI_H__ */
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index bec660357eec..2b85f87549c2 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -463,66 +463,6 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
> return 0;
> }
>
> -/*
> - * Assume that any RCIEP that emits the CXL memory expander class code
> - * is an RCD
> - */
> -static bool is_cxl_restricted(struct pci_dev *pdev)
> -{
> - return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END;
> -}
> -
> -static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
> - struct cxl_register_map *map)
> -{
> - struct cxl_port *port;
> - struct cxl_dport *dport;
> - resource_size_t component_reg_phys;
> -
> - *map = (struct cxl_register_map) {
> - .host = &pdev->dev,
> - .resource = CXL_RESOURCE_NONE,
> - };
> -
> - port = cxl_pci_find_port(pdev, &dport);
> - if (!port)
> - return -EPROBE_DEFER;
> -
> - component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport);
> -
> - put_device(&port->dev);
> -
> - if (component_reg_phys == CXL_RESOURCE_NONE)
> - return -ENXIO;
> -
> - map->resource = component_reg_phys;
> - map->reg_type = CXL_REGLOC_RBI_COMPONENT;
> - map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
> -
> - return 0;
> -}
> -
> -static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
> - struct cxl_register_map *map, u32 *caps)
> -{
> - int rc;
> -
> - rc = cxl_find_regblock(pdev, type, map);
> -
> - /*
> - * If the Register Locator DVSEC does not exist, check if it
> - * is an RCH and try to extract the Component Registers from
> - * an RCRB.
> - */
> - if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev))
> - rc = cxl_rcrb_get_comp_regs(pdev, map);
> -
> - if (rc)
> - return rc;
> -
> - return cxl_setup_regs(map, caps);
> -}
> -
> static int cxl_pci_ras_unmask(struct pci_dev *pdev)
> {
> struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 04/20] cxl: move pci generic code
2024-09-11 23:55 ` Dave Jiang
@ 2024-09-16 9:46 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 9:46 UTC (permalink / raw)
To: Dave Jiang, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/12/24 00:55, Dave Jiang wrote:
>
> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Inside cxl/core/pci.c there are helpers for CXL PCIe initialization
>> meanwhile cxl/pci.c implements the functionality for a Type3 device
>> initialization.
>>
>> Move those functions required also for Type2 initialization to
>> cxl/core/pci.c with a specific function using that moved code added in
>> a following patch.
> Please consider rephrasing as:
>
> Move helper functions from cxl/pci.c to cxl/core/pci.c in order to be
> exported and shared with CXL Type2 device initialization.
It makes sense.
I'll do.
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/pci.c | 63 ++++++++++++++++++++++++++++++++++++++++++
>> drivers/cxl/cxlpci.h | 3 ++
>> drivers/cxl/pci.c | 60 ----------------------------------------
>> 3 files changed, 66 insertions(+), 60 deletions(-)
>>
>> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
>> index 57370d9beb32..bf57f081ef8f 100644
>> --- a/drivers/cxl/core/pci.c
>> +++ b/drivers/cxl/core/pci.c
>> @@ -1079,6 +1079,69 @@ bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port)
>> }
>> EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, CXL);
>>
>> +/*
>> + * Assume that any RCIEP that emits the CXL memory expander class code
>> + * is an RCD
>> + */
>> +bool is_cxl_restricted(struct pci_dev *pdev)
>> +{
>> + return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(is_cxl_restricted, CXL);
>> +
>> +static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
>> + struct cxl_register_map *map)
>> +{
>> + struct cxl_port *port;
>> + struct cxl_dport *dport;
>> + resource_size_t component_reg_phys;
>> +
>> + *map = (struct cxl_register_map) {
>> + .host = &pdev->dev,
>> + .resource = CXL_RESOURCE_NONE,
>> + };
>> +
>> + port = cxl_pci_find_port(pdev, &dport);
>> + if (!port)
>> + return -EPROBE_DEFER;
>> +
>> + component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport);
>> +
>> + put_device(&port->dev);
>> +
>> + if (component_reg_phys == CXL_RESOURCE_NONE)
>> + return -ENXIO;
>> +
>> + map->resource = component_reg_phys;
>> + map->reg_type = CXL_REGLOC_RBI_COMPONENT;
>> + map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
>> +
>> + return 0;
>> +}
>> +
>> +int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
>> + struct cxl_register_map *map,
>> + u32 *caps)
>> +{
>> + int rc;
>> +
>> + rc = cxl_find_regblock(pdev, type, map);
>> +
>> + /*
>> + * If the Register Locator DVSEC does not exist, check if it
>> + * is an RCH and try to extract the Component Registers from
>> + * an RCRB.
>> + */
>> + if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev))
>> + rc = cxl_rcrb_get_comp_regs(pdev, map);
>> +
>> + if (rc)
>> + return rc;
>> +
>> + return cxl_setup_regs(map, caps);
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, CXL);
>> +
>> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> u32 *current_caps)
>> {
>> diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
>> index eb59019fe5f3..786b811effba 100644
>> --- a/drivers/cxl/cxlpci.h
>> +++ b/drivers/cxl/cxlpci.h
>> @@ -113,4 +113,7 @@ void read_cdat_data(struct cxl_port *port);
>> void cxl_cor_error_detected(struct pci_dev *pdev);
>> pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
>> pci_channel_state_t state);
>> +bool is_cxl_restricted(struct pci_dev *pdev);
>> +int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
>> + struct cxl_register_map *map, u32 *caps);
> Does this need to go to a different header like include/cxl/pci.h or something for type2 consumption?
>
> DJ
Next patch introduces another function for accel drivers which calls
this one, hiding things for accel drivers and improving manageability.
Thanks
>> #endif /* __CXL_PCI_H__ */
>> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
>> index bec660357eec..2b85f87549c2 100644
>> --- a/drivers/cxl/pci.c
>> +++ b/drivers/cxl/pci.c
>> @@ -463,66 +463,6 @@ static int cxl_pci_setup_mailbox(struct cxl_memdev_state *mds, bool irq_avail)
>> return 0;
>> }
>>
>> -/*
>> - * Assume that any RCIEP that emits the CXL memory expander class code
>> - * is an RCD
>> - */
>> -static bool is_cxl_restricted(struct pci_dev *pdev)
>> -{
>> - return pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_END;
>> -}
>> -
>> -static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev,
>> - struct cxl_register_map *map)
>> -{
>> - struct cxl_port *port;
>> - struct cxl_dport *dport;
>> - resource_size_t component_reg_phys;
>> -
>> - *map = (struct cxl_register_map) {
>> - .host = &pdev->dev,
>> - .resource = CXL_RESOURCE_NONE,
>> - };
>> -
>> - port = cxl_pci_find_port(pdev, &dport);
>> - if (!port)
>> - return -EPROBE_DEFER;
>> -
>> - component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport);
>> -
>> - put_device(&port->dev);
>> -
>> - if (component_reg_phys == CXL_RESOURCE_NONE)
>> - return -ENXIO;
>> -
>> - map->resource = component_reg_phys;
>> - map->reg_type = CXL_REGLOC_RBI_COMPONENT;
>> - map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE;
>> -
>> - return 0;
>> -}
>> -
>> -static int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
>> - struct cxl_register_map *map, u32 *caps)
>> -{
>> - int rc;
>> -
>> - rc = cxl_find_regblock(pdev, type, map);
>> -
>> - /*
>> - * If the Register Locator DVSEC does not exist, check if it
>> - * is an RCH and try to extract the Component Registers from
>> - * an RCRB.
>> - */
>> - if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev))
>> - rc = cxl_rcrb_get_comp_regs(pdev, map);
>> -
>> - if (rc)
>> - return rc;
>> -
>> - return cxl_setup_regs(map, caps);
>> -}
>> -
>> static int cxl_pci_ras_unmask(struct pci_dev *pdev)
>> {
>> struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 05/20] cxl: add function for type2 cxl regs setup
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (3 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 04/20] cxl: move pci generic code alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-10 6:00 ` Li, Ming4
2024-09-13 17:32 ` Jonathan Cameron
2024-09-07 8:18 ` [PATCH v3 06/20] cxl: add functions for resource request/release by a driver alejandro.lucero-palau
` (14 subsequent siblings)
19 siblings, 2 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Create a new function for a type2 device initialising
cxl_dev_state struct regarding cxl regs setup and mapping.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/pci.c | 30 ++++++++++++++++++++++++++++++
drivers/net/ethernet/sfc/efx_cxl.c | 6 ++++++
include/linux/cxl/cxl.h | 2 ++
3 files changed, 38 insertions(+)
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index bf57f081ef8f..9afcdd643866 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -1142,6 +1142,36 @@ int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
}
EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, CXL);
+int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds)
+{
+ struct cxl_register_map map;
+ int rc;
+
+ rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map,
+ &cxlds->capabilities);
+ if (!rc) {
+ rc = cxl_map_device_regs(&map, &cxlds->regs.device_regs);
+ if (rc)
+ return rc;
+ }
+
+ rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
+ &cxlds->reg_map, &cxlds->capabilities);
+ if (rc)
+ dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
+
+ if (cxlds->capabilities & BIT(CXL_CM_CAP_CAP_ID_RAS)) {
+ rc = cxl_map_component_regs(&cxlds->reg_map,
+ &cxlds->regs.component,
+ BIT(CXL_CM_CAP_CAP_ID_RAS));
+ if (rc)
+ dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
+ }
+
+ return rc;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_pci_accel_setup_regs, CXL);
+
bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
u32 *current_caps)
{
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
index bba36cbbab22..fee143e94c1f 100644
--- a/drivers/net/ethernet/sfc/efx_cxl.c
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -66,6 +66,12 @@ int efx_cxl_init(struct efx_nic *efx)
goto err;
}
+ rc = cxl_pci_accel_setup_regs(pci_dev, cxl->cxlds);
+ if (rc) {
+ pci_err(pci_dev, "CXL accel setup regs failed");
+ goto err;
+ }
+
return 0;
err:
kfree(cxl->cxlds);
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index 4a57bf60403d..f2dcba6cdc22 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -5,6 +5,7 @@
#define __CXL_H
#include <linux/device.h>
+#include <linux/pci.h>
enum cxl_resource {
CXL_ACCEL_RES_DPA,
@@ -50,4 +51,5 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
enum cxl_resource);
bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
u32 *current_caps);
+int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 05/20] cxl: add function for type2 cxl regs setup
2024-09-07 8:18 ` [PATCH v3 05/20] cxl: add function for type2 cxl regs setup alejandro.lucero-palau
@ 2024-09-10 6:00 ` Li, Ming4
2024-09-10 7:24 ` Alejandro Lucero Palau
2024-09-13 17:32 ` Jonathan Cameron
1 sibling, 1 reply; 88+ messages in thread
From: Li, Ming4 @ 2024-09-10 6:00 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Create a new function for a type2 device initialising
> cxl_dev_state struct regarding cxl regs setup and mapping.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/pci.c | 30 ++++++++++++++++++++++++++++++
> drivers/net/ethernet/sfc/efx_cxl.c | 6 ++++++
> include/linux/cxl/cxl.h | 2 ++
> 3 files changed, 38 insertions(+)
>
> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> index bf57f081ef8f..9afcdd643866 100644
> --- a/drivers/cxl/core/pci.c
> +++ b/drivers/cxl/core/pci.c
> @@ -1142,6 +1142,36 @@ int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
> }
> EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, CXL);
>
> +int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds)
> +{
> + struct cxl_register_map map;
> + int rc;
> +
> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map,
> + &cxlds->capabilities);
> + if (!rc) {
> + rc = cxl_map_device_regs(&map, &cxlds->regs.device_regs);
> + if (rc)
> + return rc;
> + }
> +
> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
> + &cxlds->reg_map, &cxlds->capabilities);
> + if (rc)
> + dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
> +
> + if (cxlds->capabilities & BIT(CXL_CM_CAP_CAP_ID_RAS)) {
> + rc = cxl_map_component_regs(&cxlds->reg_map,
> + &cxlds->regs.component,
> + BIT(CXL_CM_CAP_CAP_ID_RAS));
> + if (rc)
> + dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
> + }
> +
> + return rc;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_pci_accel_setup_regs, CXL);
> +
I thought this function should be implemented in efx driver, just like what cxl_pci driver does, because I think it is not a generic setup flow for all CXL type-2 devices.
> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> u32 *current_caps)
> {
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index bba36cbbab22..fee143e94c1f 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -66,6 +66,12 @@ int efx_cxl_init(struct efx_nic *efx)
> goto err;
> }
>
> + rc = cxl_pci_accel_setup_regs(pci_dev, cxl->cxlds);
> + if (rc) {
> + pci_err(pci_dev, "CXL accel setup regs failed");
> + goto err;
> + }
> +
> return 0;
> err:
> kfree(cxl->cxlds);
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index 4a57bf60403d..f2dcba6cdc22 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -5,6 +5,7 @@
> #define __CXL_H
>
> #include <linux/device.h>
> +#include <linux/pci.h>
>
> enum cxl_resource {
> CXL_ACCEL_RES_DPA,
> @@ -50,4 +51,5 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> enum cxl_resource);
> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> u32 *current_caps);
> +int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 05/20] cxl: add function for type2 cxl regs setup
2024-09-10 6:00 ` Li, Ming4
@ 2024-09-10 7:24 ` Alejandro Lucero Palau
2024-09-12 9:08 ` Zhi Wang
0 siblings, 1 reply; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-10 7:24 UTC (permalink / raw)
To: Li, Ming4, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/10/24 07:00, Li, Ming4 wrote:
> On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Create a new function for a type2 device initialising
>> cxl_dev_state struct regarding cxl regs setup and mapping.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/pci.c | 30 ++++++++++++++++++++++++++++++
>> drivers/net/ethernet/sfc/efx_cxl.c | 6 ++++++
>> include/linux/cxl/cxl.h | 2 ++
>> 3 files changed, 38 insertions(+)
>>
>> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
>> index bf57f081ef8f..9afcdd643866 100644
>> --- a/drivers/cxl/core/pci.c
>> +++ b/drivers/cxl/core/pci.c
>> @@ -1142,6 +1142,36 @@ int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
>> }
>> EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, CXL);
>>
>> +int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds)
>> +{
>> + struct cxl_register_map map;
>> + int rc;
>> +
>> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map,
>> + &cxlds->capabilities);
>> + if (!rc) {
>> + rc = cxl_map_device_regs(&map, &cxlds->regs.device_regs);
>> + if (rc)
>> + return rc;
>> + }
>> +
>> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
>> + &cxlds->reg_map, &cxlds->capabilities);
>> + if (rc)
>> + dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
>> +
>> + if (cxlds->capabilities & BIT(CXL_CM_CAP_CAP_ID_RAS)) {
>> + rc = cxl_map_component_regs(&cxlds->reg_map,
>> + &cxlds->regs.component,
>> + BIT(CXL_CM_CAP_CAP_ID_RAS));
>> + if (rc)
>> + dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
>> + }
>> +
>> + return rc;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_pci_accel_setup_regs, CXL);
>> +
> I thought this function should be implemented in efx driver, just like what cxl_pci driver does, because I think it is not a generic setup flow for all CXL type-2 devices.
>
The idea here is to have a single function for discovering the
registers, both Device and Component registers. If an accel has not all
of them, as in the sfc case, not a problem with the last changes added.
Keeping with the idea of avoiding an accel driver to manipulate
cxl_dev_state, this accessor is created.
>> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> u32 *current_caps)
>> {
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
>> index bba36cbbab22..fee143e94c1f 100644
>> --- a/drivers/net/ethernet/sfc/efx_cxl.c
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -66,6 +66,12 @@ int efx_cxl_init(struct efx_nic *efx)
>> goto err;
>> }
>>
>> + rc = cxl_pci_accel_setup_regs(pci_dev, cxl->cxlds);
>> + if (rc) {
>> + pci_err(pci_dev, "CXL accel setup regs failed");
>> + goto err;
>> + }
>> +
>> return 0;
>> err:
>> kfree(cxl->cxlds);
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index 4a57bf60403d..f2dcba6cdc22 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -5,6 +5,7 @@
>> #define __CXL_H
>>
>> #include <linux/device.h>
>> +#include <linux/pci.h>
>>
>> enum cxl_resource {
>> CXL_ACCEL_RES_DPA,
>> @@ -50,4 +51,5 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> enum cxl_resource);
>> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> u32 *current_caps);
>> +int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
>> #endif
>
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 05/20] cxl: add function for type2 cxl regs setup
2024-09-10 7:24 ` Alejandro Lucero Palau
@ 2024-09-12 9:08 ` Zhi Wang
0 siblings, 0 replies; 88+ messages in thread
From: Zhi Wang @ 2024-09-12 9:08 UTC (permalink / raw)
To: Alejandro Lucero Palau
Cc: Li, Ming4, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On Tue, 10 Sep 2024 08:24:33 +0100
Alejandro Lucero Palau <alucerop@amd.com> wrote:
>
> On 9/10/24 07:00, Li, Ming4 wrote:
> > On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
> >> From: Alejandro Lucero <alucerop@amd.com>
> >>
> >> Create a new function for a type2 device initialising
> >> cxl_dev_state struct regarding cxl regs setup and mapping.
> >>
> >> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> >> ---
> >> drivers/cxl/core/pci.c | 30
> >> ++++++++++++++++++++++++++++++ drivers/net/ethernet/sfc/efx_cxl.c
> >> | 6 ++++++ include/linux/cxl/cxl.h | 2 ++
> >> 3 files changed, 38 insertions(+)
> >>
> >> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> >> index bf57f081ef8f..9afcdd643866 100644
> >> --- a/drivers/cxl/core/pci.c
> >> +++ b/drivers/cxl/core/pci.c
> >> @@ -1142,6 +1142,36 @@ int cxl_pci_setup_regs(struct pci_dev
> >> *pdev, enum cxl_regloc_type type, }
> >> EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, CXL);
> >>
> >> +int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct
> >> cxl_dev_state *cxlds) +{
> >> + struct cxl_register_map map;
> >> + int rc;
> >> +
> >> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map,
> >> + &cxlds->capabilities);
> >> + if (!rc) {
> >> + rc = cxl_map_device_regs(&map,
> >> &cxlds->regs.device_regs);
> >> + if (rc)
> >> + return rc;
> >> + }
> >> +
> >> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
> >> + &cxlds->reg_map,
> >> &cxlds->capabilities);
> >> + if (rc)
> >> + dev_warn(&pdev->dev, "No component registers
> >> (%d)\n", rc); +
> >> + if (cxlds->capabilities & BIT(CXL_CM_CAP_CAP_ID_RAS)) {
> >> + rc = cxl_map_component_regs(&cxlds->reg_map,
> >> +
> >> &cxlds->regs.component,
> >> +
> >> BIT(CXL_CM_CAP_CAP_ID_RAS));
> >> + if (rc)
> >> + dev_dbg(&pdev->dev, "Failed to map RAS
> >> capability.\n");
> >> + }
> >> +
> >> + return rc;
> >> +}
> >> +EXPORT_SYMBOL_NS_GPL(cxl_pci_accel_setup_regs, CXL);
> >> +
> > I thought this function should be implemented in efx driver, just
> > like what cxl_pci driver does, because I think it is not a generic
> > setup flow for all CXL type-2 devices.
> >
>
> The idea here is to have a single function for discovering the
> registers, both Device and Component registers. If an accel has not
> all of them, as in the sfc case, not a problem with the last changes
> added.
>
> Keeping with the idea of avoiding an accel driver to manipulate
> cxl_dev_state, this accessor is created.
>
Agree. Let's keep this function.
>
> >> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32
> >> expected_caps, u32 *current_caps)
> >> {
> >> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c
> >> b/drivers/net/ethernet/sfc/efx_cxl.c index
> >> bba36cbbab22..fee143e94c1f 100644 ---
> >> a/drivers/net/ethernet/sfc/efx_cxl.c +++
> >> b/drivers/net/ethernet/sfc/efx_cxl.c @@ -66,6 +66,12 @@ int
> >> efx_cxl_init(struct efx_nic *efx) goto err;
> >> }
> >>
> >> + rc = cxl_pci_accel_setup_regs(pci_dev, cxl->cxlds);
> >> + if (rc) {
> >> + pci_err(pci_dev, "CXL accel setup regs failed");
> >> + goto err;
> >> + }
> >> +
> >> return 0;
> >> err:
> >> kfree(cxl->cxlds);
> >> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> >> index 4a57bf60403d..f2dcba6cdc22 100644
> >> --- a/include/linux/cxl/cxl.h
> >> +++ b/include/linux/cxl/cxl.h
> >> @@ -5,6 +5,7 @@
> >> #define __CXL_H
> >>
> >> #include <linux/device.h>
> >> +#include <linux/pci.h>
> >>
> >> enum cxl_resource {
> >> CXL_ACCEL_RES_DPA,
> >> @@ -50,4 +51,5 @@ int cxl_set_resource(struct cxl_dev_state
> >> *cxlds, struct resource res, enum cxl_resource);
> >> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32
> >> expected_caps, u32 *current_caps);
> >> +int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct
> >> cxl_dev_state *cxlds); #endif
> >
>
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 05/20] cxl: add function for type2 cxl regs setup
2024-09-07 8:18 ` [PATCH v3 05/20] cxl: add function for type2 cxl regs setup alejandro.lucero-palau
2024-09-10 6:00 ` Li, Ming4
@ 2024-09-13 17:32 ` Jonathan Cameron
2024-09-16 12:23 ` Alejandro Lucero Palau
1 sibling, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 17:32 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:21 +0100
<alejandro.lucero-palau@amd.com> wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Create a new function for a type2 device initialising
> cxl_dev_state struct regarding cxl regs setup and mapping.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/pci.c | 30 ++++++++++++++++++++++++++++++
> drivers/net/ethernet/sfc/efx_cxl.c | 6 ++++++
> include/linux/cxl/cxl.h | 2 ++
> 3 files changed, 38 insertions(+)
>
> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> index bf57f081ef8f..9afcdd643866 100644
> --- a/drivers/cxl/core/pci.c
> +++ b/drivers/cxl/core/pci.c
> @@ -1142,6 +1142,36 @@ int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
> }
> EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, CXL);
>
> +int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds)
> +{
> + struct cxl_register_map map;
> + int rc;
> +
> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map,
> + &cxlds->capabilities);
> + if (!rc) {
I'd be tempted to wrap these two up in a separate function called
from here as the out of line good path is less than ideal from
readability point of view.
> + rc = cxl_map_device_regs(&map, &cxlds->regs.device_regs);
> + if (rc)
> + return rc;
> + }
> +
> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
> + &cxlds->reg_map, &cxlds->capabilities);
> + if (rc)
> + dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
> +
> + if (cxlds->capabilities & BIT(CXL_CM_CAP_CAP_ID_RAS)) {
If there are no component registers this isn't going to work yet this
tries anyway?
> + rc = cxl_map_component_regs(&cxlds->reg_map,
> + &cxlds->regs.component,
> + BIT(CXL_CM_CAP_CAP_ID_RAS));
> + if (rc)
> + dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
> + }
> +
> + return rc;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_pci_accel_setup_regs, CXL);
> +
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 05/20] cxl: add function for type2 cxl regs setup
2024-09-13 17:32 ` Jonathan Cameron
@ 2024-09-16 12:23 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 12:23 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/13/24 18:32, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:21 +0100
> <alejandro.lucero-palau@amd.com> wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Create a new function for a type2 device initialising
>> cxl_dev_state struct regarding cxl regs setup and mapping.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/pci.c | 30 ++++++++++++++++++++++++++++++
>> drivers/net/ethernet/sfc/efx_cxl.c | 6 ++++++
>> include/linux/cxl/cxl.h | 2 ++
>> 3 files changed, 38 insertions(+)
>>
>> diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
>> index bf57f081ef8f..9afcdd643866 100644
>> --- a/drivers/cxl/core/pci.c
>> +++ b/drivers/cxl/core/pci.c
>> @@ -1142,6 +1142,36 @@ int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
>> }
>> EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, CXL);
>>
>> +int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds)
>> +{
>> + struct cxl_register_map map;
>> + int rc;
>> +
>> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_MEMDEV, &map,
>> + &cxlds->capabilities);
>> + if (!rc) {
> I'd be tempted to wrap these two up in a separate function called
> from here as the out of line good path is less than ideal from
> readability point of view.
OK. It makes sense.
>> + rc = cxl_map_device_regs(&map, &cxlds->regs.device_regs);
>> + if (rc)
>> + return rc;
>> + }
>> +
>> + rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT,
>> + &cxlds->reg_map, &cxlds->capabilities);
>> + if (rc)
>> + dev_warn(&pdev->dev, "No component registers (%d)\n", rc);
>> +
>> + if (cxlds->capabilities & BIT(CXL_CM_CAP_CAP_ID_RAS)) {
> If there are no component registers this isn't going to work yet this
> tries anyway?
Right. I should return if no component registers.
Thanks!
>> + rc = cxl_map_component_regs(&cxlds->reg_map,
>> + &cxlds->regs.component,
>> + BIT(CXL_CM_CAP_CAP_ID_RAS));
>> + if (rc)
>> + dev_dbg(&pdev->dev, "Failed to map RAS capability.\n");
>> + }
>> +
>> + return rc;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_pci_accel_setup_regs, CXL);
>> +
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 06/20] cxl: add functions for resource request/release by a driver
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (4 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 05/20] cxl: add function for type2 cxl regs setup alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-10 6:15 ` Li, Ming4
2024-09-13 17:35 ` Jonathan Cameron
2024-09-07 8:18 ` [PATCH v3 07/20] cxl: harden resource_contains checks to handle zero size resources alejandro.lucero-palau
` (13 subsequent siblings)
19 siblings, 2 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Create accessors for an accel driver requesting and
releaseing a resource.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/memdev.c | 40 ++++++++++++++++++++++++++++++
drivers/net/ethernet/sfc/efx_cxl.c | 7 ++++++
include/linux/cxl/cxl.h | 2 ++
3 files changed, 49 insertions(+)
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 10c0a6990f9a..a7d8daf4a59b 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -744,6 +744,46 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
}
EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
+int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
+{
+ int rc;
+
+ switch (type) {
+ case CXL_ACCEL_RES_RAM:
+ rc = request_resource(&cxlds->dpa_res, &cxlds->ram_res);
+ break;
+ case CXL_ACCEL_RES_PMEM:
+ rc = request_resource(&cxlds->dpa_res, &cxlds->pmem_res);
+ break;
+ default:
+ dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
+ return -EINVAL;
+ }
+
+ return rc;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_request_resource, CXL);
+
+int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
+{
+ int rc;
+
+ switch (type) {
+ case CXL_ACCEL_RES_RAM:
+ rc = release_resource(&cxlds->ram_res);
+ break;
+ case CXL_ACCEL_RES_PMEM:
+ rc = release_resource(&cxlds->pmem_res);
+ break;
+ default:
+ dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
+ return -EINVAL;
+ }
+
+ return rc;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_release_resource, CXL);
+
static int cxl_memdev_release_file(struct inode *inode, struct file *file)
{
struct cxl_memdev *cxlmd =
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
index fee143e94c1f..80259c8317fd 100644
--- a/drivers/net/ethernet/sfc/efx_cxl.c
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -72,6 +72,12 @@ int efx_cxl_init(struct efx_nic *efx)
goto err;
}
+ rc = cxl_request_resource(cxl->cxlds, CXL_ACCEL_RES_RAM);
+ if (rc) {
+ pci_err(pci_dev, "CXL request resource failed");
+ goto err;
+ }
+
return 0;
err:
kfree(cxl->cxlds);
@@ -84,6 +90,7 @@ int efx_cxl_init(struct efx_nic *efx)
void efx_cxl_exit(struct efx_nic *efx)
{
if (efx->cxl) {
+ cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
kfree(efx->cxl->cxlds);
kfree(efx->cxl);
}
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index f2dcba6cdc22..22912b2d9bb2 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -52,4 +52,6 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
u32 *current_caps);
int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
+int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
+int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 06/20] cxl: add functions for resource request/release by a driver
2024-09-07 8:18 ` [PATCH v3 06/20] cxl: add functions for resource request/release by a driver alejandro.lucero-palau
@ 2024-09-10 6:15 ` Li, Ming4
2024-09-16 8:15 ` Alejandro Lucero Palau
2024-09-13 17:35 ` Jonathan Cameron
1 sibling, 1 reply; 88+ messages in thread
From: Li, Ming4 @ 2024-09-10 6:15 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Create accessors for an accel driver requesting and
> releaseing a resource.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/memdev.c | 40 ++++++++++++++++++++++++++++++
> drivers/net/ethernet/sfc/efx_cxl.c | 7 ++++++
> include/linux/cxl/cxl.h | 2 ++
> 3 files changed, 49 insertions(+)
>
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 10c0a6990f9a..a7d8daf4a59b 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -744,6 +744,46 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> }
> EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
>
> +int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
> +{
> + int rc;
> +
> + switch (type) {
> + case CXL_ACCEL_RES_RAM:
Should check ram_res size before request_resource().
> + rc = request_resource(&cxlds->dpa_res, &cxlds->ram_res);
> + break;
> + case CXL_ACCEL_RES_PMEM:
> + rc = request_resource(&cxlds->dpa_res, &cxlds->pmem_res);
Same as above. Checking the size of pmem_res.
> + break;
> + default:
> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
> + return -EINVAL;
> + }
> +
> + return rc;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_request_resource, CXL);
> +
> +int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
> +{
> + int rc;
> +
> + switch (type) {
> + case CXL_ACCEL_RES_RAM:
> + rc = release_resource(&cxlds->ram_res);
> + break;
> + case CXL_ACCEL_RES_PMEM:
> + rc = release_resource(&cxlds->pmem_res);
> + break;
> + default:
> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
> + return -EINVAL;
> + }
> +
> + return rc;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_release_resource, CXL);
> +
> static int cxl_memdev_release_file(struct inode *inode, struct file *file)
> {
> struct cxl_memdev *cxlmd =
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index fee143e94c1f..80259c8317fd 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -72,6 +72,12 @@ int efx_cxl_init(struct efx_nic *efx)
> goto err;
> }
>
> + rc = cxl_request_resource(cxl->cxlds, CXL_ACCEL_RES_RAM);
> + if (rc) {
> + pci_err(pci_dev, "CXL request resource failed");
> + goto err;
> + }
> +
> return 0;
> err:
> kfree(cxl->cxlds);
> @@ -84,6 +90,7 @@ int efx_cxl_init(struct efx_nic *efx)
> void efx_cxl_exit(struct efx_nic *efx)
> {
> if (efx->cxl) {
> + cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
> kfree(efx->cxl->cxlds);
> kfree(efx->cxl);
> }
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index f2dcba6cdc22..22912b2d9bb2 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -52,4 +52,6 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> u32 *current_caps);
> int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
> +int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
> +int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 06/20] cxl: add functions for resource request/release by a driver
2024-09-10 6:15 ` Li, Ming4
@ 2024-09-16 8:15 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 8:15 UTC (permalink / raw)
To: Li, Ming4, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/10/24 07:15, Li, Ming4 wrote:
> On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Create accessors for an accel driver requesting and
>> releaseing a resource.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/memdev.c | 40 ++++++++++++++++++++++++++++++
>> drivers/net/ethernet/sfc/efx_cxl.c | 7 ++++++
>> include/linux/cxl/cxl.h | 2 ++
>> 3 files changed, 49 insertions(+)
>>
>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>> index 10c0a6990f9a..a7d8daf4a59b 100644
>> --- a/drivers/cxl/core/memdev.c
>> +++ b/drivers/cxl/core/memdev.c
>> @@ -744,6 +744,46 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> }
>> EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
>>
>> +int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
>> +{
>> + int rc;
>> +
>> + switch (type) {
>> + case CXL_ACCEL_RES_RAM:
> Should check ram_res size before request_resource().
I'll do.
>> + rc = request_resource(&cxlds->dpa_res, &cxlds->ram_res);
>> + break;
>> + case CXL_ACCEL_RES_PMEM:
>> + rc = request_resource(&cxlds->dpa_res, &cxlds->pmem_res);
> Same as above. Checking the size of pmem_res.
I'll do.
Thanks
>> + break;
>> + default:
>> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
>> + return -EINVAL;
>> + }
>> +
>> + return rc;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_request_resource, CXL);
>> +
>> +int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
>> +{
>> + int rc;
>> +
>> + switch (type) {
>> + case CXL_ACCEL_RES_RAM:
>> + rc = release_resource(&cxlds->ram_res);
>> + break;
>> + case CXL_ACCEL_RES_PMEM:
>> + rc = release_resource(&cxlds->pmem_res);
>> + break;
>> + default:
>> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
>> + return -EINVAL;
>> + }
>> +
>> + return rc;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_release_resource, CXL);
>> +
>> static int cxl_memdev_release_file(struct inode *inode, struct file *file)
>> {
>> struct cxl_memdev *cxlmd =
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
>> index fee143e94c1f..80259c8317fd 100644
>> --- a/drivers/net/ethernet/sfc/efx_cxl.c
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -72,6 +72,12 @@ int efx_cxl_init(struct efx_nic *efx)
>> goto err;
>> }
>>
>> + rc = cxl_request_resource(cxl->cxlds, CXL_ACCEL_RES_RAM);
>> + if (rc) {
>> + pci_err(pci_dev, "CXL request resource failed");
>> + goto err;
>> + }
>> +
>> return 0;
>> err:
>> kfree(cxl->cxlds);
>> @@ -84,6 +90,7 @@ int efx_cxl_init(struct efx_nic *efx)
>> void efx_cxl_exit(struct efx_nic *efx)
>> {
>> if (efx->cxl) {
>> + cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
>> kfree(efx->cxl->cxlds);
>> kfree(efx->cxl);
>> }
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index f2dcba6cdc22..22912b2d9bb2 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -52,4 +52,6 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> u32 *current_caps);
>> int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
>> +int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
>> +int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
>> #endif
>
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 06/20] cxl: add functions for resource request/release by a driver
2024-09-07 8:18 ` [PATCH v3 06/20] cxl: add functions for resource request/release by a driver alejandro.lucero-palau
2024-09-10 6:15 ` Li, Ming4
@ 2024-09-13 17:35 ` Jonathan Cameron
2024-09-16 12:33 ` Alejandro Lucero Palau
1 sibling, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 17:35 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:22 +0100
alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Create accessors for an accel driver requesting and
> releaseing a resource.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/memdev.c | 40 ++++++++++++++++++++++++++++++
> drivers/net/ethernet/sfc/efx_cxl.c | 7 ++++++
> include/linux/cxl/cxl.h | 2 ++
> 3 files changed, 49 insertions(+)
>
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 10c0a6990f9a..a7d8daf4a59b 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -744,6 +744,46 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> }
> EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
>
> +int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
> +{
> + int rc;
> +
> + switch (type) {
> + case CXL_ACCEL_RES_RAM:
> + rc = request_resource(&cxlds->dpa_res, &cxlds->ram_res);
> + break;
> + case CXL_ACCEL_RES_PMEM:
> + rc = request_resource(&cxlds->dpa_res, &cxlds->pmem_res);
> + break;
return request_resource()
> + default:
> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
No unknown. We know exactly what it is (DPA) but we don't have it.
Unexpected maybe?
> + return -EINVAL;
> + }
> +
> + return rc;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_request_resource, CXL);
> +
> +int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
> +{
> + int rc;
> +
> + switch (type) {
> + case CXL_ACCEL_RES_RAM:
> + rc = release_resource(&cxlds->ram_res);
> + break;
> + case CXL_ACCEL_RES_PMEM:
> + rc = release_resource(&cxlds->pmem_res);
return ..
> + break;
> + default:
> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
As above. Probably know what we got, it it unexpected not unknown.
> + return -EINVAL;
> + }
> +
> + return rc;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_release_resource, CXL);
> +
> static int cxl_memdev_release_file(struct inode *inode, struct file *file)
> {
> struct cxl_memdev *cxlmd =
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index fee143e94c1f..80259c8317fd 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -72,6 +72,12 @@ int efx_cxl_init(struct efx_nic *efx)
> goto err;
> }
>
> + rc = cxl_request_resource(cxl->cxlds, CXL_ACCEL_RES_RAM);
> + if (rc) {
> + pci_err(pci_dev, "CXL request resource failed");
> + goto err;
> + }
> +
> return 0;
> err:
> kfree(cxl->cxlds);
> @@ -84,6 +90,7 @@ int efx_cxl_init(struct efx_nic *efx)
> void efx_cxl_exit(struct efx_nic *efx)
> {
> if (efx->cxl) {
> + cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
> kfree(efx->cxl->cxlds);
> kfree(efx->cxl);
> }
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index f2dcba6cdc22..22912b2d9bb2 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -52,4 +52,6 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
> u32 *current_caps);
> int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
> +int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
> +int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 06/20] cxl: add functions for resource request/release by a driver
2024-09-13 17:35 ` Jonathan Cameron
@ 2024-09-16 12:33 ` Alejandro Lucero Palau
2024-09-16 13:21 ` Jonathan Cameron
0 siblings, 1 reply; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 12:33 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/13/24 18:35, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:22 +0100
> alejandro.lucero-palau@amd.com wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Create accessors for an accel driver requesting and
>> releaseing a resource.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/memdev.c | 40 ++++++++++++++++++++++++++++++
>> drivers/net/ethernet/sfc/efx_cxl.c | 7 ++++++
>> include/linux/cxl/cxl.h | 2 ++
>> 3 files changed, 49 insertions(+)
>>
>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>> index 10c0a6990f9a..a7d8daf4a59b 100644
>> --- a/drivers/cxl/core/memdev.c
>> +++ b/drivers/cxl/core/memdev.c
>> @@ -744,6 +744,46 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> }
>> EXPORT_SYMBOL_NS_GPL(cxl_set_resource, CXL);
>>
>> +int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
>> +{
>> + int rc;
>> +
>> + switch (type) {
>> + case CXL_ACCEL_RES_RAM:
>> + rc = request_resource(&cxlds->dpa_res, &cxlds->ram_res);
>> + break;
>> + case CXL_ACCEL_RES_PMEM:
>> + rc = request_resource(&cxlds->dpa_res, &cxlds->pmem_res);
>> + break;
> return request_resource()
Yes.
>> + default:
>> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
> No unknown. We know exactly what it is (DPA) but we don't have it.
> Unexpected maybe?
Is this not the same case that you brought in previously? Should I keep
the default?
>> + return -EINVAL;
>> + }
>> +
>> + return rc;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_request_resource, CXL);
>> +
>> +int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
>> +{
>> + int rc;
>> +
>> + switch (type) {
>> + case CXL_ACCEL_RES_RAM:
>> + rc = release_resource(&cxlds->ram_res);
>> + break;
>> + case CXL_ACCEL_RES_PMEM:
>> + rc = release_resource(&cxlds->pmem_res);
> return ..
Sure.
Thanks
>> + break;
>> + default:
>> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
> As above. Probably know what we got, it it unexpected not unknown.
>
>> + return -EINVAL;
>> + }
>> +
>> + return rc;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_release_resource, CXL);
>> +
>> static int cxl_memdev_release_file(struct inode *inode, struct file *file)
>> {
>> struct cxl_memdev *cxlmd =
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
>> index fee143e94c1f..80259c8317fd 100644
>> --- a/drivers/net/ethernet/sfc/efx_cxl.c
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -72,6 +72,12 @@ int efx_cxl_init(struct efx_nic *efx)
>> goto err;
>> }
>>
>> + rc = cxl_request_resource(cxl->cxlds, CXL_ACCEL_RES_RAM);
>> + if (rc) {
>> + pci_err(pci_dev, "CXL request resource failed");
>> + goto err;
>> + }
>> +
>> return 0;
>> err:
>> kfree(cxl->cxlds);
>> @@ -84,6 +90,7 @@ int efx_cxl_init(struct efx_nic *efx)
>> void efx_cxl_exit(struct efx_nic *efx)
>> {
>> if (efx->cxl) {
>> + cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
>> kfree(efx->cxl->cxlds);
>> kfree(efx->cxl);
>> }
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index f2dcba6cdc22..22912b2d9bb2 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -52,4 +52,6 @@ int cxl_set_resource(struct cxl_dev_state *cxlds, struct resource res,
>> bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
>> u32 *current_caps);
>> int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
>> +int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
>> +int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
>> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 06/20] cxl: add functions for resource request/release by a driver
2024-09-16 12:33 ` Alejandro Lucero Palau
@ 2024-09-16 13:21 ` Jonathan Cameron
0 siblings, 0 replies; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-16 13:21 UTC (permalink / raw)
To: Alejandro Lucero Palau
Cc: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
>
> >> + default:
> >> + dev_err(cxlds->dev, "unknown resource type (%u)\n", type);
> > No unknown. We know exactly what it is (DPA) but we don't have it.
> > Unexpected maybe?
>
>
> Is this not the same case that you brought in previously? Should I keep
> the default?
Just change the message to "unsupported resource type..."
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 07/20] cxl: harden resource_contains checks to handle zero size resources
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (5 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 06/20] cxl: add functions for resource request/release by a driver alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-13 17:36 ` Jonathan Cameron
2024-09-07 8:18 ` [PATCH v3 08/20] cxl: add function for setting media ready by a driver alejandro.lucero-palau
` (12 subsequent siblings)
19 siblings, 1 reply; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
For a resource defined with size zero, resource_contains returns
always true.
Add resource size check before using it.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/hdm.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
index 3df10517a327..953a5f86a43f 100644
--- a/drivers/cxl/core/hdm.c
+++ b/drivers/cxl/core/hdm.c
@@ -327,10 +327,11 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
cxled->dpa_res = res;
cxled->skip = skipped;
- if (resource_contains(&cxlds->pmem_res, res))
+ if ((resource_size(&cxlds->pmem_res)) && (resource_contains(&cxlds->pmem_res, res))) {
cxled->mode = CXL_DECODER_PMEM;
- else if (resource_contains(&cxlds->ram_res, res))
+ } else if ((resource_size(&cxlds->ram_res)) && (resource_contains(&cxlds->ram_res, res))) {
cxled->mode = CXL_DECODER_RAM;
+ }
else {
dev_warn(dev, "decoder%d.%d: %pr mixed mode not supported\n",
port->id, cxled->cxld.id, cxled->dpa_res);
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 07/20] cxl: harden resource_contains checks to handle zero size resources
2024-09-07 8:18 ` [PATCH v3 07/20] cxl: harden resource_contains checks to handle zero size resources alejandro.lucero-palau
@ 2024-09-13 17:36 ` Jonathan Cameron
2024-09-16 12:36 ` Alejandro Lucero Palau
0 siblings, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 17:36 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:23 +0100
<alejandro.lucero-palau@amd.com> wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> For a resource defined with size zero, resource_contains returns
> always true.
>
> Add resource size check before using it.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/hdm.c | 5 +++--
> 1 file changed, 3 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
> index 3df10517a327..953a5f86a43f 100644
> --- a/drivers/cxl/core/hdm.c
> +++ b/drivers/cxl/core/hdm.c
> @@ -327,10 +327,11 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
> cxled->dpa_res = res;
> cxled->skip = skipped;
>
> - if (resource_contains(&cxlds->pmem_res, res))
> + if ((resource_size(&cxlds->pmem_res)) && (resource_contains(&cxlds->pmem_res, res))) {
Excess brackets + I'd break that over two lines.
> cxled->mode = CXL_DECODER_PMEM;
> - else if (resource_contains(&cxlds->ram_res, res))
> + } else if ((resource_size(&cxlds->ram_res)) && (resource_contains(&cxlds->ram_res, res))) {
Same here,
> cxled->mode = CXL_DECODER_RAM;
> + }
> else {
> dev_warn(dev, "decoder%d.%d: %pr mixed mode not supported\n",
> port->id, cxled->cxld.id, cxled->dpa_res);
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 07/20] cxl: harden resource_contains checks to handle zero size resources
2024-09-13 17:36 ` Jonathan Cameron
@ 2024-09-16 12:36 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 12:36 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/13/24 18:36, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:23 +0100
> <alejandro.lucero-palau@amd.com> wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> For a resource defined with size zero, resource_contains returns
>> always true.
>>
>> Add resource size check before using it.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/hdm.c | 5 +++--
>> 1 file changed, 3 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
>> index 3df10517a327..953a5f86a43f 100644
>> --- a/drivers/cxl/core/hdm.c
>> +++ b/drivers/cxl/core/hdm.c
>> @@ -327,10 +327,11 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
>> cxled->dpa_res = res;
>> cxled->skip = skipped;
>>
>> - if (resource_contains(&cxlds->pmem_res, res))
>> + if ((resource_size(&cxlds->pmem_res)) && (resource_contains(&cxlds->pmem_res, res))) {
> Excess brackets + I'd break that over two lines.
Right. I'll fix it.
Thanks
>
>> cxled->mode = CXL_DECODER_PMEM;
>> - else if (resource_contains(&cxlds->ram_res, res))
>> + } else if ((resource_size(&cxlds->ram_res)) && (resource_contains(&cxlds->ram_res, res))) {
> Same here,
>
>> cxled->mode = CXL_DECODER_RAM;
>> + }
>> else {
>> dev_warn(dev, "decoder%d.%d: %pr mixed mode not supported\n",
>> port->id, cxled->cxld.id, cxled->dpa_res);
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 08/20] cxl: add function for setting media ready by a driver
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (6 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 07/20] cxl: harden resource_contains checks to handle zero size resources alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-07 8:18 ` [PATCH v3 09/20] cxl: support type2 memdev creation alejandro.lucero-palau
` (11 subsequent siblings)
19 siblings, 0 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
A Type-2 driver can require to set the memory availability explicitly.
Add a function to the exported CXL API for accelerator drivers.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/memdev.c | 6 ++++++
drivers/net/ethernet/sfc/efx_cxl.c | 5 +++++
include/linux/cxl/cxl.h | 1 +
3 files changed, 12 insertions(+)
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index a7d8daf4a59b..836faf09b328 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -784,6 +784,12 @@ int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type)
}
EXPORT_SYMBOL_NS_GPL(cxl_release_resource, CXL);
+void cxl_set_media_ready(struct cxl_dev_state *cxlds)
+{
+ cxlds->media_ready = true;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_set_media_ready, CXL);
+
static int cxl_memdev_release_file(struct inode *inode, struct file *file)
{
struct cxl_memdev *cxlmd =
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
index 80259c8317fd..14fab41fe10a 100644
--- a/drivers/net/ethernet/sfc/efx_cxl.c
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -78,6 +78,11 @@ int efx_cxl_init(struct efx_nic *efx)
goto err;
}
+ /* We do not have the register about media status. Hardware design
+ * implies it is ready.
+ */
+ cxl_set_media_ready(cxl->cxlds);
+
return 0;
err:
kfree(cxl->cxlds);
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index 22912b2d9bb2..08723b2d75bc 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -54,4 +54,5 @@ bool cxl_pci_check_caps(struct cxl_dev_state *cxlds, u32 expected_caps,
int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
+void cxl_set_media_ready(struct cxl_dev_state *cxlds);
#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* [PATCH v3 09/20] cxl: support type2 memdev creation
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (7 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 08/20] cxl: add function for setting media ready by a driver alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-12 18:19 ` Dave Jiang
2024-09-07 8:18 ` [PATCH v3 10/20] cxl: indicate probe deferral alejandro.lucero-palau
` (10 subsequent siblings)
19 siblings, 1 reply; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Add memdev creation from sfc driver.
Current cxl core is relying on a CXL_DEVTYPE_CLASSMEM type device when
creating a memdev leading to problems when obtaining cxl_memdev_state
references from a CXL_DEVTYPE_DEVMEM type. This last device type is
managed by a specific vendor driver and does not need same sysfs files
since not userspace intervention is expected. This patch checks for the
right device type in those functions using cxl_memdev_state.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/cdat.c | 3 +++
drivers/cxl/core/memdev.c | 9 +++++++++
drivers/cxl/mem.c | 17 +++++++++++------
drivers/net/ethernet/sfc/efx_cxl.c | 7 +++++++
include/linux/cxl/cxl.h | 2 ++
5 files changed, 32 insertions(+), 6 deletions(-)
diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c
index bb83867d9fec..0d4679c137d4 100644
--- a/drivers/cxl/core/cdat.c
+++ b/drivers/cxl/core/cdat.c
@@ -558,6 +558,9 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr,
};
struct cxl_dpa_perf *perf;
+ if (!mds)
+ return;
+
switch (cxlr->mode) {
case CXL_DECODER_RAM:
perf = &mds->ram_perf;
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 836faf09b328..5f8418620b70 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -468,6 +468,9 @@ static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ if (!mds)
+ return 0;
+
if (a == &dev_attr_ram_qos_class.attr)
if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID)
return 0;
@@ -487,6 +490,9 @@ static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ if (!mds)
+ return 0;
+
if (a == &dev_attr_pmem_qos_class.attr)
if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID)
return 0;
@@ -507,6 +513,9 @@ static umode_t cxl_memdev_security_visible(struct kobject *kobj,
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ if (!mds)
+ return 0;
+
if (a == &dev_attr_security_sanitize.attr &&
!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
return 0;
diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
index 7de232eaeb17..5c7ad230bccb 100644
--- a/drivers/cxl/mem.c
+++ b/drivers/cxl/mem.c
@@ -131,12 +131,14 @@ static int cxl_mem_probe(struct device *dev)
dentry = cxl_debugfs_create_dir(dev_name(dev));
debugfs_create_devm_seqfile(dev, "dpamem", dentry, cxl_mem_dpa_show);
- if (test_bit(CXL_POISON_ENABLED_INJECT, mds->poison.enabled_cmds))
- debugfs_create_file("inject_poison", 0200, dentry, cxlmd,
- &cxl_poison_inject_fops);
- if (test_bit(CXL_POISON_ENABLED_CLEAR, mds->poison.enabled_cmds))
- debugfs_create_file("clear_poison", 0200, dentry, cxlmd,
- &cxl_poison_clear_fops);
+ if (mds) {
+ if (test_bit(CXL_POISON_ENABLED_INJECT, mds->poison.enabled_cmds))
+ debugfs_create_file("inject_poison", 0200, dentry, cxlmd,
+ &cxl_poison_inject_fops);
+ if (test_bit(CXL_POISON_ENABLED_CLEAR, mds->poison.enabled_cmds))
+ debugfs_create_file("clear_poison", 0200, dentry, cxlmd,
+ &cxl_poison_clear_fops);
+ }
rc = devm_add_action_or_reset(dev, remove_debugfs, dentry);
if (rc)
@@ -222,6 +224,9 @@ static umode_t cxl_mem_visible(struct kobject *kobj, struct attribute *a, int n)
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
+ if (!mds)
+ return 0;
+
if (a == &dev_attr_trigger_poison_list.attr)
if (!test_bit(CXL_POISON_ENABLED_LIST,
mds->poison.enabled_cmds))
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
index 14fab41fe10a..899bc823a212 100644
--- a/drivers/net/ethernet/sfc/efx_cxl.c
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -83,6 +83,13 @@ int efx_cxl_init(struct efx_nic *efx)
*/
cxl_set_media_ready(cxl->cxlds);
+ cxl->cxlmd = devm_cxl_add_memdev(&pci_dev->dev, cxl->cxlds);
+ if (IS_ERR(cxl->cxlmd)) {
+ pci_err(pci_dev, "CXL accel memdev creation failed");
+ rc = PTR_ERR(cxl->cxlmd);
+ goto err;
+ }
+
return 0;
err:
kfree(cxl->cxlds);
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index 08723b2d75bc..fc0859f841dc 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -55,4 +55,6 @@ int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
void cxl_set_media_ready(struct cxl_dev_state *cxlds);
+struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
+ struct cxl_dev_state *cxlds);
#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 09/20] cxl: support type2 memdev creation
2024-09-07 8:18 ` [PATCH v3 09/20] cxl: support type2 memdev creation alejandro.lucero-palau
@ 2024-09-12 18:19 ` Dave Jiang
2024-09-16 12:38 ` Alejandro Lucero Palau
0 siblings, 1 reply; 88+ messages in thread
From: Dave Jiang @ 2024-09-12 18:19 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Add memdev creation from sfc driver.
>
> Current cxl core is relying on a CXL_DEVTYPE_CLASSMEM type device when
> creating a memdev leading to problems when obtaining cxl_memdev_state
> references from a CXL_DEVTYPE_DEVMEM type. This last device type is
> managed by a specific vendor driver and does not need same sysfs files
> since not userspace intervention is expected. This patch checks for the
> right device type in those functions using cxl_memdev_state.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/cdat.c | 3 +++
> drivers/cxl/core/memdev.c | 9 +++++++++
> drivers/cxl/mem.c | 17 +++++++++++------
> drivers/net/ethernet/sfc/efx_cxl.c | 7 +++++++
> include/linux/cxl/cxl.h | 2 ++
> 5 files changed, 32 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c
> index bb83867d9fec..0d4679c137d4 100644
> --- a/drivers/cxl/core/cdat.c
> +++ b/drivers/cxl/core/cdat.c
> @@ -558,6 +558,9 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr,
> };
> struct cxl_dpa_perf *perf;
>
> + if (!mds)
> + return;
> +
> switch (cxlr->mode) {
> case CXL_DECODER_RAM:
> perf = &mds->ram_perf;
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 836faf09b328..5f8418620b70 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -468,6 +468,9 @@ static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
>
> + if (!mds)
> + return 0;
I think instead of altering the sysfs visible attributes, what you really want is to not register the unwanted group attributes. Or basically only register the cxl_memdev_attribute_group and not the other ones. Otherwise it gets really messy. And whomever adds future attributes need to also remember the special case. I think you can refer to core/port.c and look at the different decoder types as inspiration for creating different memdev types where it has cxl_decoder_base_attribute_group and then tack on specific attribute groups.
DJ
> +
> if (a == &dev_attr_ram_qos_class.attr)
> if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID)
> return 0;
> @@ -487,6 +490,9 @@ static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n
> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
>
> + if (!mds)
> + return 0;
> +
> if (a == &dev_attr_pmem_qos_class.attr)
> if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID)
> return 0;
> @@ -507,6 +513,9 @@ static umode_t cxl_memdev_security_visible(struct kobject *kobj,
> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
>
> + if (!mds)
> + return 0;
> +
> if (a == &dev_attr_security_sanitize.attr &&
> !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
> return 0;
> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
> index 7de232eaeb17..5c7ad230bccb 100644
> --- a/drivers/cxl/mem.c
> +++ b/drivers/cxl/mem.c
> @@ -131,12 +131,14 @@ static int cxl_mem_probe(struct device *dev)
> dentry = cxl_debugfs_create_dir(dev_name(dev));
> debugfs_create_devm_seqfile(dev, "dpamem", dentry, cxl_mem_dpa_show);
>
> - if (test_bit(CXL_POISON_ENABLED_INJECT, mds->poison.enabled_cmds))
> - debugfs_create_file("inject_poison", 0200, dentry, cxlmd,
> - &cxl_poison_inject_fops);
> - if (test_bit(CXL_POISON_ENABLED_CLEAR, mds->poison.enabled_cmds))
> - debugfs_create_file("clear_poison", 0200, dentry, cxlmd,
> - &cxl_poison_clear_fops);
> + if (mds) {
> + if (test_bit(CXL_POISON_ENABLED_INJECT, mds->poison.enabled_cmds))
> + debugfs_create_file("inject_poison", 0200, dentry, cxlmd,
> + &cxl_poison_inject_fops);
> + if (test_bit(CXL_POISON_ENABLED_CLEAR, mds->poison.enabled_cmds))
> + debugfs_create_file("clear_poison", 0200, dentry, cxlmd,
> + &cxl_poison_clear_fops);
> + }
>
> rc = devm_add_action_or_reset(dev, remove_debugfs, dentry);
> if (rc)
> @@ -222,6 +224,9 @@ static umode_t cxl_mem_visible(struct kobject *kobj, struct attribute *a, int n)
> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
>
> + if (!mds)
> + return 0;
> +
> if (a == &dev_attr_trigger_poison_list.attr)
> if (!test_bit(CXL_POISON_ENABLED_LIST,
> mds->poison.enabled_cmds))
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index 14fab41fe10a..899bc823a212 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -83,6 +83,13 @@ int efx_cxl_init(struct efx_nic *efx)
> */
> cxl_set_media_ready(cxl->cxlds);
>
> + cxl->cxlmd = devm_cxl_add_memdev(&pci_dev->dev, cxl->cxlds);
> + if (IS_ERR(cxl->cxlmd)) {
> + pci_err(pci_dev, "CXL accel memdev creation failed");
> + rc = PTR_ERR(cxl->cxlmd);
> + goto err;
> + }
> +
> return 0;
> err:
> kfree(cxl->cxlds);
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index 08723b2d75bc..fc0859f841dc 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -55,4 +55,6 @@ int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
> int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
> int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
> void cxl_set_media_ready(struct cxl_dev_state *cxlds);
> +struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
> + struct cxl_dev_state *cxlds);
> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 09/20] cxl: support type2 memdev creation
2024-09-12 18:19 ` Dave Jiang
@ 2024-09-16 12:38 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 12:38 UTC (permalink / raw)
To: Dave Jiang, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/12/24 19:19, Dave Jiang wrote:
>
> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Add memdev creation from sfc driver.
>>
>> Current cxl core is relying on a CXL_DEVTYPE_CLASSMEM type device when
>> creating a memdev leading to problems when obtaining cxl_memdev_state
>> references from a CXL_DEVTYPE_DEVMEM type. This last device type is
>> managed by a specific vendor driver and does not need same sysfs files
>> since not userspace intervention is expected. This patch checks for the
>> right device type in those functions using cxl_memdev_state.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/cdat.c | 3 +++
>> drivers/cxl/core/memdev.c | 9 +++++++++
>> drivers/cxl/mem.c | 17 +++++++++++------
>> drivers/net/ethernet/sfc/efx_cxl.c | 7 +++++++
>> include/linux/cxl/cxl.h | 2 ++
>> 5 files changed, 32 insertions(+), 6 deletions(-)
>>
>> diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c
>> index bb83867d9fec..0d4679c137d4 100644
>> --- a/drivers/cxl/core/cdat.c
>> +++ b/drivers/cxl/core/cdat.c
>> @@ -558,6 +558,9 @@ void cxl_region_perf_data_calculate(struct cxl_region *cxlr,
>> };
>> struct cxl_dpa_perf *perf;
>>
>> + if (!mds)
>> + return;
>> +
>> switch (cxlr->mode) {
>> case CXL_DECODER_RAM:
>> perf = &mds->ram_perf;
>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>> index 836faf09b328..5f8418620b70 100644
>> --- a/drivers/cxl/core/memdev.c
>> +++ b/drivers/cxl/core/memdev.c
>> @@ -468,6 +468,9 @@ static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
>> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
>> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
>>
>> + if (!mds)
>> + return 0;
> I think instead of altering the sysfs visible attributes, what you really want is to not register the unwanted group attributes. Or basically only register the cxl_memdev_attribute_group and not the other ones. Otherwise it gets really messy. And whomever adds future attributes need to also remember the special case. I think you can refer to core/port.c and look at the different decoder types as inspiration for creating different memdev types where it has cxl_decoder_base_attribute_group and then tack on specific attribute groups.
>
> DJ
I think you are right. It was a quick fix for having things working and
not thought about it in detail.
I'll do and fix it for v4.
Thanks!
>> +
>> if (a == &dev_attr_ram_qos_class.attr)
>> if (mds->ram_perf.qos_class == CXL_QOS_CLASS_INVALID)
>> return 0;
>> @@ -487,6 +490,9 @@ static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n
>> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
>> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
>>
>> + if (!mds)
>> + return 0;
>> +
>> if (a == &dev_attr_pmem_qos_class.attr)
>> if (mds->pmem_perf.qos_class == CXL_QOS_CLASS_INVALID)
>> return 0;
>> @@ -507,6 +513,9 @@ static umode_t cxl_memdev_security_visible(struct kobject *kobj,
>> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
>> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
>>
>> + if (!mds)
>> + return 0;
>> +
>> if (a == &dev_attr_security_sanitize.attr &&
>> !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
>> return 0;
>> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
>> index 7de232eaeb17..5c7ad230bccb 100644
>> --- a/drivers/cxl/mem.c
>> +++ b/drivers/cxl/mem.c
>> @@ -131,12 +131,14 @@ static int cxl_mem_probe(struct device *dev)
>> dentry = cxl_debugfs_create_dir(dev_name(dev));
>> debugfs_create_devm_seqfile(dev, "dpamem", dentry, cxl_mem_dpa_show);
>>
>> - if (test_bit(CXL_POISON_ENABLED_INJECT, mds->poison.enabled_cmds))
>> - debugfs_create_file("inject_poison", 0200, dentry, cxlmd,
>> - &cxl_poison_inject_fops);
>> - if (test_bit(CXL_POISON_ENABLED_CLEAR, mds->poison.enabled_cmds))
>> - debugfs_create_file("clear_poison", 0200, dentry, cxlmd,
>> - &cxl_poison_clear_fops);
>> + if (mds) {
>> + if (test_bit(CXL_POISON_ENABLED_INJECT, mds->poison.enabled_cmds))
>> + debugfs_create_file("inject_poison", 0200, dentry, cxlmd,
>> + &cxl_poison_inject_fops);
>> + if (test_bit(CXL_POISON_ENABLED_CLEAR, mds->poison.enabled_cmds))
>> + debugfs_create_file("clear_poison", 0200, dentry, cxlmd,
>> + &cxl_poison_clear_fops);
>> + }
>>
>> rc = devm_add_action_or_reset(dev, remove_debugfs, dentry);
>> if (rc)
>> @@ -222,6 +224,9 @@ static umode_t cxl_mem_visible(struct kobject *kobj, struct attribute *a, int n)
>> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
>> struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
>>
>> + if (!mds)
>> + return 0;
>> +
>> if (a == &dev_attr_trigger_poison_list.attr)
>> if (!test_bit(CXL_POISON_ENABLED_LIST,
>> mds->poison.enabled_cmds))
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
>> index 14fab41fe10a..899bc823a212 100644
>> --- a/drivers/net/ethernet/sfc/efx_cxl.c
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -83,6 +83,13 @@ int efx_cxl_init(struct efx_nic *efx)
>> */
>> cxl_set_media_ready(cxl->cxlds);
>>
>> + cxl->cxlmd = devm_cxl_add_memdev(&pci_dev->dev, cxl->cxlds);
>> + if (IS_ERR(cxl->cxlmd)) {
>> + pci_err(pci_dev, "CXL accel memdev creation failed");
>> + rc = PTR_ERR(cxl->cxlmd);
>> + goto err;
>> + }
>> +
>> return 0;
>> err:
>> kfree(cxl->cxlds);
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index 08723b2d75bc..fc0859f841dc 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -55,4 +55,6 @@ int cxl_pci_accel_setup_regs(struct pci_dev *pdev, struct cxl_dev_state *cxlds);
>> int cxl_request_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
>> int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
>> void cxl_set_media_ready(struct cxl_dev_state *cxlds);
>> +struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>> + struct cxl_dev_state *cxlds);
>> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 10/20] cxl: indicate probe deferral
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (8 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 09/20] cxl: support type2 memdev creation alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-10 6:37 ` Li, Ming4
` (2 more replies)
2024-09-07 8:18 ` [PATCH v3 11/20] cxl: define a driver interface for HPA free space enumaration alejandro.lucero-palau
` (9 subsequent siblings)
19 siblings, 3 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
The first stop for a CXL accelerator driver that wants to establish new
CXL.mem regions is to register a 'struct cxl_memdev. That kicks off
cxl_mem_probe() to enumerate all 'struct cxl_port' instances in the
topology up to the root.
If the root driver has not attached yet the expectation is that the
driver waits until that link is established. The common cxl_pci_driver
has reason to keep the 'struct cxl_memdev' device attached to the bus
until the root driver attaches. An accelerator may want to instead defer
probing until CXL resources can be acquired.
Use the @endpoint attribute of a 'struct cxl_memdev' to convey when
accelerator driver probing should be deferred vs failed. Provide that
indication via a new cxl_acquire_endpoint() API that can retrieve the
probe status of the memdev.
Based on https://lore.kernel.org/linux-cxl/168592155270.1948938.11536845108449547920.stgit@dwillia2-xfh.jf.intel.com/
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
Co-developed-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/cxl/core/memdev.c | 67 +++++++++++++++++++++++++++++++++++++++
drivers/cxl/core/port.c | 2 +-
drivers/cxl/mem.c | 4 ++-
include/linux/cxl/cxl.h | 2 ++
4 files changed, 73 insertions(+), 2 deletions(-)
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index 5f8418620b70..d4406cf3ed32 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -5,6 +5,7 @@
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/firmware.h>
#include <linux/device.h>
+#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include <linux/pci.h>
@@ -23,6 +24,8 @@ static DECLARE_RWSEM(cxl_memdev_rwsem);
static int cxl_mem_major;
static DEFINE_IDA(cxl_memdev_ida);
+static unsigned short endpoint_ready_timeout = HZ;
+
static void cxl_memdev_release(struct device *dev)
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
@@ -1163,6 +1166,70 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
+/*
+ * Try to get a locked reference on a memdev's CXL port topology
+ * connection. Be careful to observe when cxl_mem_probe() has deposited
+ * a probe deferral awaiting the arrival of the CXL root driver.
+ */
+struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd)
+{
+ struct cxl_port *endpoint;
+ unsigned long timeout;
+ int rc = -ENXIO;
+
+ /*
+ * A memdev creation triggers ports creation through the kernel
+ * device object model. An endpoint port could not be created yet
+ * but coming. Wait here for a gentle space of time for ensuring
+ * and endpoint port not there is due to some error and not because
+ * the race described.
+ *
+ * Note this is a similar case this function is implemented for, but
+ * instead of the race with the root port, this is against its own
+ * endpoint port.
+ */
+ timeout = jiffies + endpoint_ready_timeout;
+ do {
+ device_lock(&cxlmd->dev);
+ endpoint = cxlmd->endpoint;
+ if (endpoint)
+ break;
+ device_unlock(&cxlmd->dev);
+ if (msleep_interruptible(100)) {
+ device_lock(&cxlmd->dev);
+ break;
+ }
+ } while (!time_after(jiffies, timeout));
+
+ if (!endpoint)
+ goto err;
+
+ if (IS_ERR(endpoint)) {
+ rc = PTR_ERR(endpoint);
+ goto err;
+ }
+
+ device_lock(&endpoint->dev);
+ if (!endpoint->dev.driver)
+ goto err_endpoint;
+
+ return endpoint;
+
+err_endpoint:
+ device_unlock(&endpoint->dev);
+err:
+ device_unlock(&cxlmd->dev);
+ return ERR_PTR(rc);
+}
+EXPORT_SYMBOL_NS(cxl_acquire_endpoint, CXL);
+
+void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
+{
+ device_unlock(&endpoint->dev);
+ device_unlock(&cxlmd->dev);
+}
+EXPORT_SYMBOL_NS(cxl_release_endpoint, CXL);
+
static void sanitize_teardown_notifier(void *data)
{
struct cxl_memdev_state *mds = data;
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 39b20ddd0296..ca2c993faa9c 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -1554,7 +1554,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
*/
dev_dbg(&cxlmd->dev, "%s is a root dport\n",
dev_name(dport_dev));
- return -ENXIO;
+ return -EPROBE_DEFER;
}
parent_port = find_cxl_port(dparent, &parent_dport);
diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
index 5c7ad230bccb..56fd7a100c2f 100644
--- a/drivers/cxl/mem.c
+++ b/drivers/cxl/mem.c
@@ -145,8 +145,10 @@ static int cxl_mem_probe(struct device *dev)
return rc;
rc = devm_cxl_enumerate_ports(cxlmd);
- if (rc)
+ if (rc) {
+ cxlmd->endpoint = ERR_PTR(rc);
return rc;
+ }
parent_port = cxl_mem_find_port(cxlmd, &dport);
if (!parent_port) {
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index fc0859f841dc..7e4580fb8659 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -57,4 +57,6 @@ int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
void cxl_set_media_ready(struct cxl_dev_state *cxlds);
struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
struct cxl_dev_state *cxlds);
+struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd);
+void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 10/20] cxl: indicate probe deferral
2024-09-07 8:18 ` [PATCH v3 10/20] cxl: indicate probe deferral alejandro.lucero-palau
@ 2024-09-10 6:37 ` Li, Ming4
2024-09-16 8:24 ` Alejandro Lucero Palau
2024-09-12 9:19 ` Zhi Wang
2024-09-13 17:43 ` Jonathan Cameron
2 siblings, 1 reply; 88+ messages in thread
From: Li, Ming4 @ 2024-09-10 6:37 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> The first stop for a CXL accelerator driver that wants to establish new
> CXL.mem regions is to register a 'struct cxl_memdev. That kicks off
> cxl_mem_probe() to enumerate all 'struct cxl_port' instances in the
> topology up to the root.
>
> If the root driver has not attached yet the expectation is that the
> driver waits until that link is established. The common cxl_pci_driver
> has reason to keep the 'struct cxl_memdev' device attached to the bus
> until the root driver attaches. An accelerator may want to instead defer
> probing until CXL resources can be acquired.
>
> Use the @endpoint attribute of a 'struct cxl_memdev' to convey when
> accelerator driver probing should be deferred vs failed. Provide that
> indication via a new cxl_acquire_endpoint() API that can retrieve the
> probe status of the memdev.
>
> Based on https://lore.kernel.org/linux-cxl/168592155270.1948938.11536845108449547920.stgit@dwillia2-xfh.jf.intel.com/
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> ---
> drivers/cxl/core/memdev.c | 67 +++++++++++++++++++++++++++++++++++++++
> drivers/cxl/core/port.c | 2 +-
> drivers/cxl/mem.c | 4 ++-
> include/linux/cxl/cxl.h | 2 ++
> 4 files changed, 73 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 5f8418620b70..d4406cf3ed32 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -5,6 +5,7 @@
> #include <linux/io-64-nonatomic-lo-hi.h>
> #include <linux/firmware.h>
> #include <linux/device.h>
> +#include <linux/delay.h>
> #include <linux/slab.h>
> #include <linux/idr.h>
> #include <linux/pci.h>
> @@ -23,6 +24,8 @@ static DECLARE_RWSEM(cxl_memdev_rwsem);
> static int cxl_mem_major;
> static DEFINE_IDA(cxl_memdev_ida);
>
> +static unsigned short endpoint_ready_timeout = HZ;
> +
> static void cxl_memdev_release(struct device *dev)
> {
> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
> @@ -1163,6 +1166,70 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
> }
> EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
>
> +/*
> + * Try to get a locked reference on a memdev's CXL port topology
> + * connection. Be careful to observe when cxl_mem_probe() has deposited
> + * a probe deferral awaiting the arrival of the CXL root driver.
> + */
> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd)
> +{
> + struct cxl_port *endpoint;
> + unsigned long timeout;
> + int rc = -ENXIO;
> +
> + /*
> + * A memdev creation triggers ports creation through the kernel
> + * device object model. An endpoint port could not be created yet
> + * but coming. Wait here for a gentle space of time for ensuring
> + * and endpoint port not there is due to some error and not because
> + * the race described.
> + *
> + * Note this is a similar case this function is implemented for, but
> + * instead of the race with the root port, this is against its own
> + * endpoint port.
> + */
> + timeout = jiffies + endpoint_ready_timeout;
> + do {
> + device_lock(&cxlmd->dev);
> + endpoint = cxlmd->endpoint;
> + if (endpoint)
> + break;
> + device_unlock(&cxlmd->dev);
> + if (msleep_interruptible(100)) {
> + device_lock(&cxlmd->dev);
> + break;
Can exit directly. not need to hold the lock of cxlmd->dev then break.
> + }
> + } while (!time_after(jiffies, timeout));
> +
> + if (!endpoint)
> + goto err;
> +
> + if (IS_ERR(endpoint)) {
> + rc = PTR_ERR(endpoint);
> + goto err;
> + }
> +
> + device_lock(&endpoint->dev);
> + if (!endpoint->dev.driver)
> + goto err_endpoint;
> +
> + return endpoint;
> +
> +err_endpoint:
> + device_unlock(&endpoint->dev);
> +err:
> + device_unlock(&cxlmd->dev);
> + return ERR_PTR(rc);
> +}
> +EXPORT_SYMBOL_NS(cxl_acquire_endpoint, CXL);
> +
> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
> +{
> + device_unlock(&endpoint->dev);
> + device_unlock(&cxlmd->dev);
> +}
> +EXPORT_SYMBOL_NS(cxl_release_endpoint, CXL);
> +
> static void sanitize_teardown_notifier(void *data)
> {
> struct cxl_memdev_state *mds = data;
> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
> index 39b20ddd0296..ca2c993faa9c 100644
> --- a/drivers/cxl/core/port.c
> +++ b/drivers/cxl/core/port.c
> @@ -1554,7 +1554,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
> */
> dev_dbg(&cxlmd->dev, "%s is a root dport\n",
> dev_name(dport_dev));
> - return -ENXIO;
> + return -EPROBE_DEFER;
> }
>
> parent_port = find_cxl_port(dparent, &parent_dport);
> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
> index 5c7ad230bccb..56fd7a100c2f 100644
> --- a/drivers/cxl/mem.c
> +++ b/drivers/cxl/mem.c
> @@ -145,8 +145,10 @@ static int cxl_mem_probe(struct device *dev)
> return rc;
>
> rc = devm_cxl_enumerate_ports(cxlmd);
> - if (rc)
> + if (rc) {
> + cxlmd->endpoint = ERR_PTR(rc);
> return rc;
> + }
>
> parent_port = cxl_mem_find_port(cxlmd, &dport);
> if (!parent_port) {
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index fc0859f841dc..7e4580fb8659 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -57,4 +57,6 @@ int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
> void cxl_set_media_ready(struct cxl_dev_state *cxlds);
> struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
> struct cxl_dev_state *cxlds);
> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd);
> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 10/20] cxl: indicate probe deferral
2024-09-10 6:37 ` Li, Ming4
@ 2024-09-16 8:24 ` Alejandro Lucero Palau
2024-09-17 3:31 ` Li, Ming4
0 siblings, 1 reply; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 8:24 UTC (permalink / raw)
To: Li, Ming4, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/10/24 07:37, Li, Ming4 wrote:
> On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> The first stop for a CXL accelerator driver that wants to establish new
>> CXL.mem regions is to register a 'struct cxl_memdev. That kicks off
>> cxl_mem_probe() to enumerate all 'struct cxl_port' instances in the
>> topology up to the root.
>>
>> If the root driver has not attached yet the expectation is that the
>> driver waits until that link is established. The common cxl_pci_driver
>> has reason to keep the 'struct cxl_memdev' device attached to the bus
>> until the root driver attaches. An accelerator may want to instead defer
>> probing until CXL resources can be acquired.
>>
>> Use the @endpoint attribute of a 'struct cxl_memdev' to convey when
>> accelerator driver probing should be deferred vs failed. Provide that
>> indication via a new cxl_acquire_endpoint() API that can retrieve the
>> probe status of the memdev.
>>
>> Based on https://lore.kernel.org/linux-cxl/168592155270.1948938.11536845108449547920.stgit@dwillia2-xfh.jf.intel.com/
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
>> ---
>> drivers/cxl/core/memdev.c | 67 +++++++++++++++++++++++++++++++++++++++
>> drivers/cxl/core/port.c | 2 +-
>> drivers/cxl/mem.c | 4 ++-
>> include/linux/cxl/cxl.h | 2 ++
>> 4 files changed, 73 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>> index 5f8418620b70..d4406cf3ed32 100644
>> --- a/drivers/cxl/core/memdev.c
>> +++ b/drivers/cxl/core/memdev.c
>> @@ -5,6 +5,7 @@
>> #include <linux/io-64-nonatomic-lo-hi.h>
>> #include <linux/firmware.h>
>> #include <linux/device.h>
>> +#include <linux/delay.h>
>> #include <linux/slab.h>
>> #include <linux/idr.h>
>> #include <linux/pci.h>
>> @@ -23,6 +24,8 @@ static DECLARE_RWSEM(cxl_memdev_rwsem);
>> static int cxl_mem_major;
>> static DEFINE_IDA(cxl_memdev_ida);
>>
>> +static unsigned short endpoint_ready_timeout = HZ;
>> +
>> static void cxl_memdev_release(struct device *dev)
>> {
>> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
>> @@ -1163,6 +1166,70 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>> }
>> EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
>>
>> +/*
>> + * Try to get a locked reference on a memdev's CXL port topology
>> + * connection. Be careful to observe when cxl_mem_probe() has deposited
>> + * a probe deferral awaiting the arrival of the CXL root driver.
>> + */
>> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd)
>> +{
>> + struct cxl_port *endpoint;
>> + unsigned long timeout;
>> + int rc = -ENXIO;
>> +
>> + /*
>> + * A memdev creation triggers ports creation through the kernel
>> + * device object model. An endpoint port could not be created yet
>> + * but coming. Wait here for a gentle space of time for ensuring
>> + * and endpoint port not there is due to some error and not because
>> + * the race described.
>> + *
>> + * Note this is a similar case this function is implemented for, but
>> + * instead of the race with the root port, this is against its own
>> + * endpoint port.
>> + */
>> + timeout = jiffies + endpoint_ready_timeout;
>> + do {
>> + device_lock(&cxlmd->dev);
>> + endpoint = cxlmd->endpoint;
>> + if (endpoint)
>> + break;
>> + device_unlock(&cxlmd->dev);
>> + if (msleep_interruptible(100)) {
>> + device_lock(&cxlmd->dev);
>> + break;
> Can exit directly. not need to hold the lock of cxlmd->dev then break.
Not sure if it is safe to do device_unlock twice, but even if so, it
looks better to my eyes to get the lock or if not to add another error path.
>
>> + }
>> + } while (!time_after(jiffies, timeout));
>> +
>> + if (!endpoint)
>> + goto err;
>> +
>> + if (IS_ERR(endpoint)) {
>> + rc = PTR_ERR(endpoint);
>> + goto err;
>> + }
>> +
>> + device_lock(&endpoint->dev);
>> + if (!endpoint->dev.driver)
>> + goto err_endpoint;
>> +
>> + return endpoint;
>> +
>> +err_endpoint:
>> + device_unlock(&endpoint->dev);
>> +err:
>> + device_unlock(&cxlmd->dev);
>> + return ERR_PTR(rc);
>> +}
>> +EXPORT_SYMBOL_NS(cxl_acquire_endpoint, CXL);
>> +
>> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
>> +{
>> + device_unlock(&endpoint->dev);
>> + device_unlock(&cxlmd->dev);
>> +}
>> +EXPORT_SYMBOL_NS(cxl_release_endpoint, CXL);
>> +
>> static void sanitize_teardown_notifier(void *data)
>> {
>> struct cxl_memdev_state *mds = data;
>> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
>> index 39b20ddd0296..ca2c993faa9c 100644
>> --- a/drivers/cxl/core/port.c
>> +++ b/drivers/cxl/core/port.c
>> @@ -1554,7 +1554,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
>> */
>> dev_dbg(&cxlmd->dev, "%s is a root dport\n",
>> dev_name(dport_dev));
>> - return -ENXIO;
>> + return -EPROBE_DEFER;
>> }
>>
>> parent_port = find_cxl_port(dparent, &parent_dport);
>> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
>> index 5c7ad230bccb..56fd7a100c2f 100644
>> --- a/drivers/cxl/mem.c
>> +++ b/drivers/cxl/mem.c
>> @@ -145,8 +145,10 @@ static int cxl_mem_probe(struct device *dev)
>> return rc;
>>
>> rc = devm_cxl_enumerate_ports(cxlmd);
>> - if (rc)
>> + if (rc) {
>> + cxlmd->endpoint = ERR_PTR(rc);
>> return rc;
>> + }
>>
>> parent_port = cxl_mem_find_port(cxlmd, &dport);
>> if (!parent_port) {
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index fc0859f841dc..7e4580fb8659 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -57,4 +57,6 @@ int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
>> void cxl_set_media_ready(struct cxl_dev_state *cxlds);
>> struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>> struct cxl_dev_state *cxlds);
>> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd);
>> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
>> #endif
>
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 10/20] cxl: indicate probe deferral
2024-09-16 8:24 ` Alejandro Lucero Palau
@ 2024-09-17 3:31 ` Li, Ming4
2024-09-17 9:16 ` Alejandro Lucero Palau
0 siblings, 1 reply; 88+ messages in thread
From: Li, Ming4 @ 2024-09-17 3:31 UTC (permalink / raw)
To: Alejandro Lucero Palau, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/16/2024 4:24 PM, Alejandro Lucero Palau wrote:
>
> On 9/10/24 07:37, Li, Ming4 wrote:
>> On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
>>> From: Alejandro Lucero <alucerop@amd.com>
>>>
>>> The first stop for a CXL accelerator driver that wants to establish new
>>> CXL.mem regions is to register a 'struct cxl_memdev. That kicks off
>>> cxl_mem_probe() to enumerate all 'struct cxl_port' instances in the
>>> topology up to the root.
>>>
>>> If the root driver has not attached yet the expectation is that the
>>> driver waits until that link is established. The common cxl_pci_driver
>>> has reason to keep the 'struct cxl_memdev' device attached to the bus
>>> until the root driver attaches. An accelerator may want to instead defer
>>> probing until CXL resources can be acquired.
>>>
>>> Use the @endpoint attribute of a 'struct cxl_memdev' to convey when
>>> accelerator driver probing should be deferred vs failed. Provide that
>>> indication via a new cxl_acquire_endpoint() API that can retrieve the
>>> probe status of the memdev.
>>>
>>> Based on https://lore.kernel.org/linux-cxl/168592155270.1948938.11536845108449547920.stgit@dwillia2-xfh.jf.intel.com/
>>>
>>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>>> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
>>> ---
>>> drivers/cxl/core/memdev.c | 67 +++++++++++++++++++++++++++++++++++++++
>>> drivers/cxl/core/port.c | 2 +-
>>> drivers/cxl/mem.c | 4 ++-
>>> include/linux/cxl/cxl.h | 2 ++
>>> 4 files changed, 73 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>>> index 5f8418620b70..d4406cf3ed32 100644
>>> --- a/drivers/cxl/core/memdev.c
>>> +++ b/drivers/cxl/core/memdev.c
>>> @@ -5,6 +5,7 @@
>>> #include <linux/io-64-nonatomic-lo-hi.h>
>>> #include <linux/firmware.h>
>>> #include <linux/device.h>
>>> +#include <linux/delay.h>
>>> #include <linux/slab.h>
>>> #include <linux/idr.h>
>>> #include <linux/pci.h>
>>> @@ -23,6 +24,8 @@ static DECLARE_RWSEM(cxl_memdev_rwsem);
>>> static int cxl_mem_major;
>>> static DEFINE_IDA(cxl_memdev_ida);
>>> +static unsigned short endpoint_ready_timeout = HZ;
>>> +
>>> static void cxl_memdev_release(struct device *dev)
>>> {
>>> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
>>> @@ -1163,6 +1166,70 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>>> }
>>> EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
>>> +/*
>>> + * Try to get a locked reference on a memdev's CXL port topology
>>> + * connection. Be careful to observe when cxl_mem_probe() has deposited
>>> + * a probe deferral awaiting the arrival of the CXL root driver.
>>> + */
>>> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd)
>>> +{
>>> + struct cxl_port *endpoint;
>>> + unsigned long timeout;
>>> + int rc = -ENXIO;
>>> +
>>> + /*
>>> + * A memdev creation triggers ports creation through the kernel
>>> + * device object model. An endpoint port could not be created yet
>>> + * but coming. Wait here for a gentle space of time for ensuring
>>> + * and endpoint port not there is due to some error and not because
>>> + * the race described.
>>> + *
>>> + * Note this is a similar case this function is implemented for, but
>>> + * instead of the race with the root port, this is against its own
>>> + * endpoint port.
>>> + */
>>> + timeout = jiffies + endpoint_ready_timeout;
>>> + do {
>>> + device_lock(&cxlmd->dev);
>>> + endpoint = cxlmd->endpoint;
>>> + if (endpoint)
>>> + break;
>>> + device_unlock(&cxlmd->dev);
>>> + if (msleep_interruptible(100)) {
>>> + device_lock(&cxlmd->dev);
>>> + break;
>> Can exit directly. not need to hold the lock of cxlmd->dev then break.
>
>
> Not sure if it is safe to do device_unlock twice, but even if so, it looks better to my eyes to get the lock or if not to add another error path.
>
why device_unlock will be called twice? directly return the value of rc like below if the sleep is interrupted.
if (msleep_interruptible(100))
return ERR_PTR(rc);
>
>
>>
>>> + }
>>> + } while (!time_after(jiffies, timeout));
Another issue I noticed is that above loop will not hold the device lock if timeout happened(without msleep interrupted), but below "goto err" will call device_unlock() for the device.
I think below 'if (!endpoint)' can also return the value of rc. Combine above changes, I think the code should be:
do {
......
if (msleep_interruptible(100))
break;
} while (!time_after(jiffies, timeout));
if (!endpoint)
return ERR_PTR(rc);
Does it make more sense?
>>> +
>>> + if (!endpoint)
>>> + goto err;
>>> +
>>> + if (IS_ERR(endpoint)) {
>>> + rc = PTR_ERR(endpoint);
>>> + goto err;
>>> + }
>>> +
>>> + device_lock(&endpoint->dev);
>>> + if (!endpoint->dev.driver)
>>> + goto err_endpoint;
>>> +
>>> + return endpoint;
>>> +
>>> +err_endpoint:
>>> + device_unlock(&endpoint->dev);
>>> +err:
>>> + device_unlock(&cxlmd->dev);
>>> + return ERR_PTR(rc);
>>> +}
>>> +EXPORT_SYMBOL_NS(cxl_acquire_endpoint, CXL);
>>> +
>>> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
>>> +{
>>> + device_unlock(&endpoint->dev);
>>> + device_unlock(&cxlmd->dev);
>>> +}
>>> +EXPORT_SYMBOL_NS(cxl_release_endpoint, CXL);
>>> +
>>> static void sanitize_teardown_notifier(void *data)
>>> {
>>> struct cxl_memdev_state *mds = data;
>>> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
>>> index 39b20ddd0296..ca2c993faa9c 100644
>>> --- a/drivers/cxl/core/port.c
>>> +++ b/drivers/cxl/core/port.c
>>> @@ -1554,7 +1554,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
>>> */
>>> dev_dbg(&cxlmd->dev, "%s is a root dport\n",
>>> dev_name(dport_dev));
>>> - return -ENXIO;
>>> + return -EPROBE_DEFER;
>>> }
>>> parent_port = find_cxl_port(dparent, &parent_dport);
>>> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
>>> index 5c7ad230bccb..56fd7a100c2f 100644
>>> --- a/drivers/cxl/mem.c
>>> +++ b/drivers/cxl/mem.c
>>> @@ -145,8 +145,10 @@ static int cxl_mem_probe(struct device *dev)
>>> return rc;
>>> rc = devm_cxl_enumerate_ports(cxlmd);
>>> - if (rc)
>>> + if (rc) {
>>> + cxlmd->endpoint = ERR_PTR(rc);
>>> return rc;
>>> + }
>>> parent_port = cxl_mem_find_port(cxlmd, &dport);
>>> if (!parent_port) {
>>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>>> index fc0859f841dc..7e4580fb8659 100644
>>> --- a/include/linux/cxl/cxl.h
>>> +++ b/include/linux/cxl/cxl.h
>>> @@ -57,4 +57,6 @@ int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
>>> void cxl_set_media_ready(struct cxl_dev_state *cxlds);
>>> struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>>> struct cxl_dev_state *cxlds);
>>> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd);
>>> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
>>> #endif
>>
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 10/20] cxl: indicate probe deferral
2024-09-17 3:31 ` Li, Ming4
@ 2024-09-17 9:16 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-17 9:16 UTC (permalink / raw)
To: Li, Ming4, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/17/24 04:31, Li, Ming4 wrote:
> On 9/16/2024 4:24 PM, Alejandro Lucero Palau wrote:
>> On 9/10/24 07:37, Li, Ming4 wrote:
>>> On 9/7/2024 4:18 PM, alejandro.lucero-palau@amd.com wrote:
>>>> From: Alejandro Lucero <alucerop@amd.com>
>>>>
>>>> The first stop for a CXL accelerator driver that wants to establish new
>>>> CXL.mem regions is to register a 'struct cxl_memdev. That kicks off
>>>> cxl_mem_probe() to enumerate all 'struct cxl_port' instances in the
>>>> topology up to the root.
>>>>
>>>> If the root driver has not attached yet the expectation is that the
>>>> driver waits until that link is established. The common cxl_pci_driver
>>>> has reason to keep the 'struct cxl_memdev' device attached to the bus
>>>> until the root driver attaches. An accelerator may want to instead defer
>>>> probing until CXL resources can be acquired.
>>>>
>>>> Use the @endpoint attribute of a 'struct cxl_memdev' to convey when
>>>> accelerator driver probing should be deferred vs failed. Provide that
>>>> indication via a new cxl_acquire_endpoint() API that can retrieve the
>>>> probe status of the memdev.
>>>>
>>>> Based on https://lore.kernel.org/linux-cxl/168592155270.1948938.11536845108449547920.stgit@dwillia2-xfh.jf.intel.com/
>>>>
>>>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>>>> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
>>>> ---
>>>> drivers/cxl/core/memdev.c | 67 +++++++++++++++++++++++++++++++++++++++
>>>> drivers/cxl/core/port.c | 2 +-
>>>> drivers/cxl/mem.c | 4 ++-
>>>> include/linux/cxl/cxl.h | 2 ++
>>>> 4 files changed, 73 insertions(+), 2 deletions(-)
>>>>
>>>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>>>> index 5f8418620b70..d4406cf3ed32 100644
>>>> --- a/drivers/cxl/core/memdev.c
>>>> +++ b/drivers/cxl/core/memdev.c
>>>> @@ -5,6 +5,7 @@
>>>> #include <linux/io-64-nonatomic-lo-hi.h>
>>>> #include <linux/firmware.h>
>>>> #include <linux/device.h>
>>>> +#include <linux/delay.h>
>>>> #include <linux/slab.h>
>>>> #include <linux/idr.h>
>>>> #include <linux/pci.h>
>>>> @@ -23,6 +24,8 @@ static DECLARE_RWSEM(cxl_memdev_rwsem);
>>>> static int cxl_mem_major;
>>>> static DEFINE_IDA(cxl_memdev_ida);
>>>> +static unsigned short endpoint_ready_timeout = HZ;
>>>> +
>>>> static void cxl_memdev_release(struct device *dev)
>>>> {
>>>> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
>>>> @@ -1163,6 +1166,70 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>>>> }
>>>> EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
>>>> +/*
>>>> + * Try to get a locked reference on a memdev's CXL port topology
>>>> + * connection. Be careful to observe when cxl_mem_probe() has deposited
>>>> + * a probe deferral awaiting the arrival of the CXL root driver.
>>>> + */
>>>> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd)
>>>> +{
>>>> + struct cxl_port *endpoint;
>>>> + unsigned long timeout;
>>>> + int rc = -ENXIO;
>>>> +
>>>> + /*
>>>> + * A memdev creation triggers ports creation through the kernel
>>>> + * device object model. An endpoint port could not be created yet
>>>> + * but coming. Wait here for a gentle space of time for ensuring
>>>> + * and endpoint port not there is due to some error and not because
>>>> + * the race described.
>>>> + *
>>>> + * Note this is a similar case this function is implemented for, but
>>>> + * instead of the race with the root port, this is against its own
>>>> + * endpoint port.
>>>> + */
>>>> + timeout = jiffies + endpoint_ready_timeout;
>>>> + do {
>>>> + device_lock(&cxlmd->dev);
>>>> + endpoint = cxlmd->endpoint;
>>>> + if (endpoint)
>>>> + break;
>>>> + device_unlock(&cxlmd->dev);
>>>> + if (msleep_interruptible(100)) {
>>>> + device_lock(&cxlmd->dev);
>>>> + break;
>>> Can exit directly. not need to hold the lock of cxlmd->dev then break.
>>
>> Not sure if it is safe to do device_unlock twice, but even if so, it looks better to my eyes to get the lock or if not to add another error path.
>>
> why device_unlock will be called twice? directly return the value of rc like below if the sleep is interrupted.
>
> if (msleep_interruptible(100))
>
> return ERR_PTR(rc);
>
>
You are right.
>>
>>>> + }
>>>> + } while (!time_after(jiffies, timeout));
> Another issue I noticed is that above loop will not hold the device lock if timeout happened(without msleep interrupted), but below "goto err" will call device_unlock() for the device.
>
> I think below 'if (!endpoint)' can also return the value of rc. Combine above changes, I think the code should be:
>
> do {
>
> ......
>
> if (msleep_interruptible(100))
>
> break;
>
> } while (!time_after(jiffies, timeout));
>
> if (!endpoint)
>
> return ERR_PTR(rc);
>
>
> Does it make more sense?
Right again.
I can see it now.
Thank you!
>
>>>> +
>>>> + if (!endpoint)
>>>> + goto err;
>>>> +
>>>> + if (IS_ERR(endpoint)) {
>>>> + rc = PTR_ERR(endpoint);
>>>> + goto err;
>>>> + }
>>>> +
>>>> + device_lock(&endpoint->dev);
>>>> + if (!endpoint->dev.driver)
>>>> + goto err_endpoint;
>>>> +
>>>> + return endpoint;
>>>> +
>>>> +err_endpoint:
>>>> + device_unlock(&endpoint->dev);
>>>> +err:
>>>> + device_unlock(&cxlmd->dev);
>>>> + return ERR_PTR(rc);
>>>> +}
>>>> +EXPORT_SYMBOL_NS(cxl_acquire_endpoint, CXL);
>>>> +
>>>> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
>>>> +{
>>>> + device_unlock(&endpoint->dev);
>>>> + device_unlock(&cxlmd->dev);
>>>> +}
>>>> +EXPORT_SYMBOL_NS(cxl_release_endpoint, CXL);
>>>> +
>>>> static void sanitize_teardown_notifier(void *data)
>>>> {
>>>> struct cxl_memdev_state *mds = data;
>>>> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
>>>> index 39b20ddd0296..ca2c993faa9c 100644
>>>> --- a/drivers/cxl/core/port.c
>>>> +++ b/drivers/cxl/core/port.c
>>>> @@ -1554,7 +1554,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
>>>> */
>>>> dev_dbg(&cxlmd->dev, "%s is a root dport\n",
>>>> dev_name(dport_dev));
>>>> - return -ENXIO;
>>>> + return -EPROBE_DEFER;
>>>> }
>>>> parent_port = find_cxl_port(dparent, &parent_dport);
>>>> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
>>>> index 5c7ad230bccb..56fd7a100c2f 100644
>>>> --- a/drivers/cxl/mem.c
>>>> +++ b/drivers/cxl/mem.c
>>>> @@ -145,8 +145,10 @@ static int cxl_mem_probe(struct device *dev)
>>>> return rc;
>>>> rc = devm_cxl_enumerate_ports(cxlmd);
>>>> - if (rc)
>>>> + if (rc) {
>>>> + cxlmd->endpoint = ERR_PTR(rc);
>>>> return rc;
>>>> + }
>>>> parent_port = cxl_mem_find_port(cxlmd, &dport);
>>>> if (!parent_port) {
>>>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>>>> index fc0859f841dc..7e4580fb8659 100644
>>>> --- a/include/linux/cxl/cxl.h
>>>> +++ b/include/linux/cxl/cxl.h
>>>> @@ -57,4 +57,6 @@ int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
>>>> void cxl_set_media_ready(struct cxl_dev_state *cxlds);
>>>> struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>>>> struct cxl_dev_state *cxlds);
>>>> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd);
>>>> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
>>>> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 10/20] cxl: indicate probe deferral
2024-09-07 8:18 ` [PATCH v3 10/20] cxl: indicate probe deferral alejandro.lucero-palau
2024-09-10 6:37 ` Li, Ming4
@ 2024-09-12 9:19 ` Zhi Wang
2024-09-16 10:08 ` Alejandro Lucero Palau
2024-09-13 17:43 ` Jonathan Cameron
2 siblings, 1 reply; 88+ messages in thread
From: Zhi Wang @ 2024-09-12 9:19 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:26 +0100
<alejandro.lucero-palau@amd.com> wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
Hi Alejandro:
When working with V2, I noticed that if CONFIG_CXL_MEM=m and cxl_mem.ko
is not loaded, loading the type-2 driver would fail on
cxl_acquire_endpoint(). Not sure if you met the same problem.
Now we are waiting for it to be loaded, it seems not ideal with the
problem.
Thanks,
Zhi.
> The first stop for a CXL accelerator driver that wants to establish
> new CXL.mem regions is to register a 'struct cxl_memdev. That kicks
> off cxl_mem_probe() to enumerate all 'struct cxl_port' instances in
> the topology up to the root.
>
> If the root driver has not attached yet the expectation is that the
> driver waits until that link is established. The common cxl_pci_driver
> has reason to keep the 'struct cxl_memdev' device attached to the bus
> until the root driver attaches. An accelerator may want to instead
> defer probing until CXL resources can be acquired.
>
> Use the @endpoint attribute of a 'struct cxl_memdev' to convey when
> accelerator driver probing should be deferred vs failed. Provide that
> indication via a new cxl_acquire_endpoint() API that can retrieve the
> probe status of the memdev.
>
> Based on
> https://lore.kernel.org/linux-cxl/168592155270.1948938.11536845108449547920.stgit@dwillia2-xfh.jf.intel.com/
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> ---
> drivers/cxl/core/memdev.c | 67
> +++++++++++++++++++++++++++++++++++++++ drivers/cxl/core/port.c |
> 2 +- drivers/cxl/mem.c | 4 ++-
> include/linux/cxl/cxl.h | 2 ++
> 4 files changed, 73 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 5f8418620b70..d4406cf3ed32 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -5,6 +5,7 @@
> #include <linux/io-64-nonatomic-lo-hi.h>
> #include <linux/firmware.h>
> #include <linux/device.h>
> +#include <linux/delay.h>
> #include <linux/slab.h>
> #include <linux/idr.h>
> #include <linux/pci.h>
> @@ -23,6 +24,8 @@ static DECLARE_RWSEM(cxl_memdev_rwsem);
> static int cxl_mem_major;
> static DEFINE_IDA(cxl_memdev_ida);
>
> +static unsigned short endpoint_ready_timeout = HZ;
> +
> static void cxl_memdev_release(struct device *dev)
> {
> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
> @@ -1163,6 +1166,70 @@ struct cxl_memdev *devm_cxl_add_memdev(struct
> device *host, }
> EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
>
> +/*
> + * Try to get a locked reference on a memdev's CXL port topology
> + * connection. Be careful to observe when cxl_mem_probe() has
> deposited
> + * a probe deferral awaiting the arrival of the CXL root driver.
> + */
> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd)
> +{
> + struct cxl_port *endpoint;
> + unsigned long timeout;
> + int rc = -ENXIO;
> +
> + /*
> + * A memdev creation triggers ports creation through the
> kernel
> + * device object model. An endpoint port could not be
> created yet
> + * but coming. Wait here for a gentle space of time for
> ensuring
> + * and endpoint port not there is due to some error and not
> because
> + * the race described.
> + *
> + * Note this is a similar case this function is implemented
> for, but
> + * instead of the race with the root port, this is against
> its own
> + * endpoint port.
> + */
> + timeout = jiffies + endpoint_ready_timeout;
> + do {
> + device_lock(&cxlmd->dev);
> + endpoint = cxlmd->endpoint;
> + if (endpoint)
> + break;
> + device_unlock(&cxlmd->dev);
> + if (msleep_interruptible(100)) {
> + device_lock(&cxlmd->dev);
> + break;
> + }
> + } while (!time_after(jiffies, timeout));
> +
> + if (!endpoint)
> + goto err;
> +
> + if (IS_ERR(endpoint)) {
> + rc = PTR_ERR(endpoint);
> + goto err;
> + }
> +
> + device_lock(&endpoint->dev);
> + if (!endpoint->dev.driver)
> + goto err_endpoint;
> +
> + return endpoint;
> +
> +err_endpoint:
> + device_unlock(&endpoint->dev);
> +err:
> + device_unlock(&cxlmd->dev);
> + return ERR_PTR(rc);
> +}
> +EXPORT_SYMBOL_NS(cxl_acquire_endpoint, CXL);
> +
> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port
> *endpoint) +{
> + device_unlock(&endpoint->dev);
> + device_unlock(&cxlmd->dev);
> +}
> +EXPORT_SYMBOL_NS(cxl_release_endpoint, CXL);
> +
> static void sanitize_teardown_notifier(void *data)
> {
> struct cxl_memdev_state *mds = data;
> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
> index 39b20ddd0296..ca2c993faa9c 100644
> --- a/drivers/cxl/core/port.c
> +++ b/drivers/cxl/core/port.c
> @@ -1554,7 +1554,7 @@ static int add_port_attach_ep(struct cxl_memdev
> *cxlmd, */
> dev_dbg(&cxlmd->dev, "%s is a root dport\n",
> dev_name(dport_dev));
> - return -ENXIO;
> + return -EPROBE_DEFER;
> }
>
> parent_port = find_cxl_port(dparent, &parent_dport);
> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
> index 5c7ad230bccb..56fd7a100c2f 100644
> --- a/drivers/cxl/mem.c
> +++ b/drivers/cxl/mem.c
> @@ -145,8 +145,10 @@ static int cxl_mem_probe(struct device *dev)
> return rc;
>
> rc = devm_cxl_enumerate_ports(cxlmd);
> - if (rc)
> + if (rc) {
> + cxlmd->endpoint = ERR_PTR(rc);
> return rc;
> + }
>
> parent_port = cxl_mem_find_port(cxlmd, &dport);
> if (!parent_port) {
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index fc0859f841dc..7e4580fb8659 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -57,4 +57,6 @@ int cxl_release_resource(struct cxl_dev_state
> *cxlds, enum cxl_resource type); void cxl_set_media_ready(struct
> cxl_dev_state *cxlds); struct cxl_memdev *devm_cxl_add_memdev(struct
> device *host, struct cxl_dev_state *cxlds);
> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd);
> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port
> *endpoint); #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 10/20] cxl: indicate probe deferral
2024-09-12 9:19 ` Zhi Wang
@ 2024-09-16 10:08 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 10:08 UTC (permalink / raw)
To: Zhi Wang, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/12/24 10:19, Zhi Wang wrote:
> On Sat, 7 Sep 2024 09:18:26 +0100
> <alejandro.lucero-palau@amd.com> wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
> Hi Alejandro:
>
> When working with V2, I noticed that if CONFIG_CXL_MEM=m and cxl_mem.ko
> is not loaded, loading the type-2 driver would fail on
> cxl_acquire_endpoint(). Not sure if you met the same problem.
I think I have some problems with kernel build depending on if CXL code
is configured as modules, and even if CXL is not configured at all, what
it was raised by the kernel build robot.
I'll work on this for v4.
Thanks!
> Now we are waiting for it to be loaded, it seems not ideal with the
> problem.
>
> Thanks,
> Zhi.
>
>> The first stop for a CXL accelerator driver that wants to establish
>> new CXL.mem regions is to register a 'struct cxl_memdev. That kicks
>> off cxl_mem_probe() to enumerate all 'struct cxl_port' instances in
>> the topology up to the root.
>>
>> If the root driver has not attached yet the expectation is that the
>> driver waits until that link is established. The common cxl_pci_driver
>> has reason to keep the 'struct cxl_memdev' device attached to the bus
>> until the root driver attaches. An accelerator may want to instead
>> defer probing until CXL resources can be acquired.
>>
>> Use the @endpoint attribute of a 'struct cxl_memdev' to convey when
>> accelerator driver probing should be deferred vs failed. Provide that
>> indication via a new cxl_acquire_endpoint() API that can retrieve the
>> probe status of the memdev.
>>
>> Based on
>> https://lore.kernel.org/linux-cxl/168592155270.1948938.11536845108449547920.stgit@dwillia2-xfh.jf.intel.com/
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
>> ---
>> drivers/cxl/core/memdev.c | 67
>> +++++++++++++++++++++++++++++++++++++++ drivers/cxl/core/port.c |
>> 2 +- drivers/cxl/mem.c | 4 ++-
>> include/linux/cxl/cxl.h | 2 ++
>> 4 files changed, 73 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>> index 5f8418620b70..d4406cf3ed32 100644
>> --- a/drivers/cxl/core/memdev.c
>> +++ b/drivers/cxl/core/memdev.c
>> @@ -5,6 +5,7 @@
>> #include <linux/io-64-nonatomic-lo-hi.h>
>> #include <linux/firmware.h>
>> #include <linux/device.h>
>> +#include <linux/delay.h>
>> #include <linux/slab.h>
>> #include <linux/idr.h>
>> #include <linux/pci.h>
>> @@ -23,6 +24,8 @@ static DECLARE_RWSEM(cxl_memdev_rwsem);
>> static int cxl_mem_major;
>> static DEFINE_IDA(cxl_memdev_ida);
>>
>> +static unsigned short endpoint_ready_timeout = HZ;
>> +
>> static void cxl_memdev_release(struct device *dev)
>> {
>> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
>> @@ -1163,6 +1166,70 @@ struct cxl_memdev *devm_cxl_add_memdev(struct
>> device *host, }
>> EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
>>
>> +/*
>> + * Try to get a locked reference on a memdev's CXL port topology
>> + * connection. Be careful to observe when cxl_mem_probe() has
>> deposited
>> + * a probe deferral awaiting the arrival of the CXL root driver.
>> + */
>> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd)
>> +{
>> + struct cxl_port *endpoint;
>> + unsigned long timeout;
>> + int rc = -ENXIO;
>> +
>> + /*
>> + * A memdev creation triggers ports creation through the
>> kernel
>> + * device object model. An endpoint port could not be
>> created yet
>> + * but coming. Wait here for a gentle space of time for
>> ensuring
>> + * and endpoint port not there is due to some error and not
>> because
>> + * the race described.
>> + *
>> + * Note this is a similar case this function is implemented
>> for, but
>> + * instead of the race with the root port, this is against
>> its own
>> + * endpoint port.
>> + */
>> + timeout = jiffies + endpoint_ready_timeout;
>> + do {
>> + device_lock(&cxlmd->dev);
>> + endpoint = cxlmd->endpoint;
>> + if (endpoint)
>> + break;
>> + device_unlock(&cxlmd->dev);
>> + if (msleep_interruptible(100)) {
>> + device_lock(&cxlmd->dev);
>> + break;
>> + }
>> + } while (!time_after(jiffies, timeout));
>> +
>> + if (!endpoint)
>> + goto err;
>> +
>> + if (IS_ERR(endpoint)) {
>> + rc = PTR_ERR(endpoint);
>> + goto err;
>> + }
>> +
>> + device_lock(&endpoint->dev);
>> + if (!endpoint->dev.driver)
>> + goto err_endpoint;
>> +
>> + return endpoint;
>> +
>> +err_endpoint:
>> + device_unlock(&endpoint->dev);
>> +err:
>> + device_unlock(&cxlmd->dev);
>> + return ERR_PTR(rc);
>> +}
>> +EXPORT_SYMBOL_NS(cxl_acquire_endpoint, CXL);
>> +
>> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port
>> *endpoint) +{
>> + device_unlock(&endpoint->dev);
>> + device_unlock(&cxlmd->dev);
>> +}
>> +EXPORT_SYMBOL_NS(cxl_release_endpoint, CXL);
>> +
>> static void sanitize_teardown_notifier(void *data)
>> {
>> struct cxl_memdev_state *mds = data;
>> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
>> index 39b20ddd0296..ca2c993faa9c 100644
>> --- a/drivers/cxl/core/port.c
>> +++ b/drivers/cxl/core/port.c
>> @@ -1554,7 +1554,7 @@ static int add_port_attach_ep(struct cxl_memdev
>> *cxlmd, */
>> dev_dbg(&cxlmd->dev, "%s is a root dport\n",
>> dev_name(dport_dev));
>> - return -ENXIO;
>> + return -EPROBE_DEFER;
>> }
>>
>> parent_port = find_cxl_port(dparent, &parent_dport);
>> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
>> index 5c7ad230bccb..56fd7a100c2f 100644
>> --- a/drivers/cxl/mem.c
>> +++ b/drivers/cxl/mem.c
>> @@ -145,8 +145,10 @@ static int cxl_mem_probe(struct device *dev)
>> return rc;
>>
>> rc = devm_cxl_enumerate_ports(cxlmd);
>> - if (rc)
>> + if (rc) {
>> + cxlmd->endpoint = ERR_PTR(rc);
>> return rc;
>> + }
>>
>> parent_port = cxl_mem_find_port(cxlmd, &dport);
>> if (!parent_port) {
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index fc0859f841dc..7e4580fb8659 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -57,4 +57,6 @@ int cxl_release_resource(struct cxl_dev_state
>> *cxlds, enum cxl_resource type); void cxl_set_media_ready(struct
>> cxl_dev_state *cxlds); struct cxl_memdev *devm_cxl_add_memdev(struct
>> device *host, struct cxl_dev_state *cxlds);
>> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd);
>> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port
>> *endpoint); #endif
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 10/20] cxl: indicate probe deferral
2024-09-07 8:18 ` [PATCH v3 10/20] cxl: indicate probe deferral alejandro.lucero-palau
2024-09-10 6:37 ` Li, Ming4
2024-09-12 9:19 ` Zhi Wang
@ 2024-09-13 17:43 ` Jonathan Cameron
2024-09-16 13:24 ` Alejandro Lucero Palau
2 siblings, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 17:43 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:26 +0100
alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> The first stop for a CXL accelerator driver that wants to establish new
> CXL.mem regions is to register a 'struct cxl_memdev. That kicks off
> cxl_mem_probe() to enumerate all 'struct cxl_port' instances in the
> topology up to the root.
>
> If the root driver has not attached yet the expectation is that the
> driver waits until that link is established. The common cxl_pci_driver
> has reason to keep the 'struct cxl_memdev' device attached to the bus
> until the root driver attaches. An accelerator may want to instead defer
> probing until CXL resources can be acquired.
>
> Use the @endpoint attribute of a 'struct cxl_memdev' to convey when
> accelerator driver probing should be deferred vs failed. Provide that
> indication via a new cxl_acquire_endpoint() API that can retrieve the
> probe status of the memdev.
>
> Based on https://lore.kernel.org/linux-cxl/168592155270.1948938.11536845108449547920.stgit@dwillia2-xfh.jf.intel.com/
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> ---
> drivers/cxl/core/memdev.c | 67 +++++++++++++++++++++++++++++++++++++++
> drivers/cxl/core/port.c | 2 +-
> drivers/cxl/mem.c | 4 ++-
> include/linux/cxl/cxl.h | 2 ++
> 4 files changed, 73 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
> index 5f8418620b70..d4406cf3ed32 100644
> --- a/drivers/cxl/core/memdev.c
> +++ b/drivers/cxl/core/memdev.c
> @@ -5,6 +5,7 @@
> #include <linux/io-64-nonatomic-lo-hi.h>
> #include <linux/firmware.h>
> #include <linux/device.h>
> +#include <linux/delay.h>
> #include <linux/slab.h>
> #include <linux/idr.h>
> #include <linux/pci.h>
> @@ -23,6 +24,8 @@ static DECLARE_RWSEM(cxl_memdev_rwsem);
> static int cxl_mem_major;
> static DEFINE_IDA(cxl_memdev_ida);
>
> +static unsigned short endpoint_ready_timeout = HZ;
> +
> static void cxl_memdev_release(struct device *dev)
> {
> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
> @@ -1163,6 +1166,70 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
> }
> EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
>
> +/*
> + * Try to get a locked reference on a memdev's CXL port topology
> + * connection. Be careful to observe when cxl_mem_probe() has deposited
> + * a probe deferral awaiting the arrival of the CXL root driver.
> + */
> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd)
> +{
> + struct cxl_port *endpoint;
> + unsigned long timeout;
> + int rc = -ENXIO;
> +
> + /*
> + * A memdev creation triggers ports creation through the kernel
> + * device object model. An endpoint port could not be created yet
> + * but coming. Wait here for a gentle space of time for ensuring
> + * and endpoint port not there is due to some error and not because
> + * the race described.
> + *
> + * Note this is a similar case this function is implemented for, but
> + * instead of the race with the root port, this is against its own
> + * endpoint port.
This dance is nasty and there is no real guarantee it will even help.
We need a better solution. I'm not quite sure on what it is though.
Is there any precedence for similar 'wait a bit and hope'
in the kernel?
> + */
> + timeout = jiffies + endpoint_ready_timeout;
> + do {
> + device_lock(&cxlmd->dev);
> + endpoint = cxlmd->endpoint;
> + if (endpoint)
> + break;
> + device_unlock(&cxlmd->dev);
> + if (msleep_interruptible(100)) {
> + device_lock(&cxlmd->dev);
> + break;
> + }
> + } while (!time_after(jiffies, timeout));
> +
> + if (!endpoint)
> + goto err;
> +
> + if (IS_ERR(endpoint)) {
> + rc = PTR_ERR(endpoint);
> + goto err;
> + }
> +
> + device_lock(&endpoint->dev);
> + if (!endpoint->dev.driver)
> + goto err_endpoint;
> +
> + return endpoint;
> +
> +err_endpoint:
> + device_unlock(&endpoint->dev);
> +err:
> + device_unlock(&cxlmd->dev);
> + return ERR_PTR(rc);
> +}
> +EXPORT_SYMBOL_NS(cxl_acquire_endpoint, CXL);
> +
> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
> +{
> + device_unlock(&endpoint->dev);
> + device_unlock(&cxlmd->dev);
> +}
> +EXPORT_SYMBOL_NS(cxl_release_endpoint, CXL);
> +
> static void sanitize_teardown_notifier(void *data)
> {
> struct cxl_memdev_state *mds = data;
> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
> index 39b20ddd0296..ca2c993faa9c 100644
> --- a/drivers/cxl/core/port.c
> +++ b/drivers/cxl/core/port.c
> @@ -1554,7 +1554,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
> */
> dev_dbg(&cxlmd->dev, "%s is a root dport\n",
> dev_name(dport_dev));
> - return -ENXIO;
> + return -EPROBE_DEFER;
> }
>
> parent_port = find_cxl_port(dparent, &parent_dport);
> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
> index 5c7ad230bccb..56fd7a100c2f 100644
> --- a/drivers/cxl/mem.c
> +++ b/drivers/cxl/mem.c
> @@ -145,8 +145,10 @@ static int cxl_mem_probe(struct device *dev)
> return rc;
>
> rc = devm_cxl_enumerate_ports(cxlmd);
> - if (rc)
> + if (rc) {
> + cxlmd->endpoint = ERR_PTR(rc);
> return rc;
> + }
>
> parent_port = cxl_mem_find_port(cxlmd, &dport);
> if (!parent_port) {
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index fc0859f841dc..7e4580fb8659 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -57,4 +57,6 @@ int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
> void cxl_set_media_ready(struct cxl_dev_state *cxlds);
> struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
> struct cxl_dev_state *cxlds);
> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd);
> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 10/20] cxl: indicate probe deferral
2024-09-13 17:43 ` Jonathan Cameron
@ 2024-09-16 13:24 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 13:24 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/13/24 18:43, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:26 +0100
> alejandro.lucero-palau@amd.com wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> The first stop for a CXL accelerator driver that wants to establish new
>> CXL.mem regions is to register a 'struct cxl_memdev. That kicks off
>> cxl_mem_probe() to enumerate all 'struct cxl_port' instances in the
>> topology up to the root.
>>
>> If the root driver has not attached yet the expectation is that the
>> driver waits until that link is established. The common cxl_pci_driver
>> has reason to keep the 'struct cxl_memdev' device attached to the bus
>> until the root driver attaches. An accelerator may want to instead defer
>> probing until CXL resources can be acquired.
>>
>> Use the @endpoint attribute of a 'struct cxl_memdev' to convey when
>> accelerator driver probing should be deferred vs failed. Provide that
>> indication via a new cxl_acquire_endpoint() API that can retrieve the
>> probe status of the memdev.
>>
>> Based on https://lore.kernel.org/linux-cxl/168592155270.1948938.11536845108449547920.stgit@dwillia2-xfh.jf.intel.com/
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
>> ---
>> drivers/cxl/core/memdev.c | 67 +++++++++++++++++++++++++++++++++++++++
>> drivers/cxl/core/port.c | 2 +-
>> drivers/cxl/mem.c | 4 ++-
>> include/linux/cxl/cxl.h | 2 ++
>> 4 files changed, 73 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
>> index 5f8418620b70..d4406cf3ed32 100644
>> --- a/drivers/cxl/core/memdev.c
>> +++ b/drivers/cxl/core/memdev.c
>> @@ -5,6 +5,7 @@
>> #include <linux/io-64-nonatomic-lo-hi.h>
>> #include <linux/firmware.h>
>> #include <linux/device.h>
>> +#include <linux/delay.h>
>> #include <linux/slab.h>
>> #include <linux/idr.h>
>> #include <linux/pci.h>
>> @@ -23,6 +24,8 @@ static DECLARE_RWSEM(cxl_memdev_rwsem);
>> static int cxl_mem_major;
>> static DEFINE_IDA(cxl_memdev_ida);
>>
>> +static unsigned short endpoint_ready_timeout = HZ;
>> +
>> static void cxl_memdev_release(struct device *dev)
>> {
>> struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
>> @@ -1163,6 +1166,70 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>> }
>> EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, CXL);
>>
>> +/*
>> + * Try to get a locked reference on a memdev's CXL port topology
>> + * connection. Be careful to observe when cxl_mem_probe() has deposited
>> + * a probe deferral awaiting the arrival of the CXL root driver.
>> + */
>> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd)
>> +{
>> + struct cxl_port *endpoint;
>> + unsigned long timeout;
>> + int rc = -ENXIO;
>> +
>> + /*
>> + * A memdev creation triggers ports creation through the kernel
>> + * device object model. An endpoint port could not be created yet
>> + * but coming. Wait here for a gentle space of time for ensuring
>> + * and endpoint port not there is due to some error and not because
>> + * the race described.
>> + *
>> + * Note this is a similar case this function is implemented for, but
>> + * instead of the race with the root port, this is against its own
>> + * endpoint port.
> This dance is nasty and there is no real guarantee it will even help.
With all due respect, we know Dan's dancing credentials. Which are
yours? ... :-)
So, I found this when testing. The driver calls devm_cxl_add_memdev and
then cxl_acquire_endpoint, and the endpoint is not there yet.
Interestingly, I did not suffer it initially, so I do not know what
makes it trigger, but I have just tested it again and if I remove the
iteration with the timeout, the calls fails (it seems it does
deterministically). This is with a testing/developing VM with no other
main processing at the time of driver binding, and with 8 cores
available. Of course, if the driver does the second call without
interruption, it is unlikely another core can handle the bus
notification and create the endpoint faster. But I wonder why this is
(was) not always happening.
I agree the timeout could not be enough in some situations. I think it
all depends on when bus_notify is invoked, but I have not dug deeper.
>
> We need a better solution. I'm not quite sure on what it is though.
>
> Is there any precedence for similar 'wait a bit and hope'
> in the kernel?
You put it in a way that makes me miserable ... but maybe you are right,
since there is no certainty it will be done after the timeout, but it is
a full second ...
I will dig a bit ...
Thanks
>
>> + */
>> + timeout = jiffies + endpoint_ready_timeout;
>> + do {
>> + device_lock(&cxlmd->dev);
>> + endpoint = cxlmd->endpoint;
>> + if (endpoint)
>> + break;
>> + device_unlock(&cxlmd->dev);
>> + if (msleep_interruptible(100)) {
>> + device_lock(&cxlmd->dev);
>> + break;
>> + }
>> + } while (!time_after(jiffies, timeout));
>> +
>> + if (!endpoint)
>> + goto err;
>> +
>> + if (IS_ERR(endpoint)) {
>> + rc = PTR_ERR(endpoint);
>> + goto err;
>> + }
>> +
>> + device_lock(&endpoint->dev);
>> + if (!endpoint->dev.driver)
>> + goto err_endpoint;
>> +
>> + return endpoint;
>> +
>> +err_endpoint:
>> + device_unlock(&endpoint->dev);
>> +err:
>> + device_unlock(&cxlmd->dev);
>> + return ERR_PTR(rc);
>> +}
>> +EXPORT_SYMBOL_NS(cxl_acquire_endpoint, CXL);
>> +
>> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
>> +{
>> + device_unlock(&endpoint->dev);
>> + device_unlock(&cxlmd->dev);
>> +}
>> +EXPORT_SYMBOL_NS(cxl_release_endpoint, CXL);
>> +
>> static void sanitize_teardown_notifier(void *data)
>> {
>> struct cxl_memdev_state *mds = data;
>> diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
>> index 39b20ddd0296..ca2c993faa9c 100644
>> --- a/drivers/cxl/core/port.c
>> +++ b/drivers/cxl/core/port.c
>> @@ -1554,7 +1554,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
>> */
>> dev_dbg(&cxlmd->dev, "%s is a root dport\n",
>> dev_name(dport_dev));
>> - return -ENXIO;
>> + return -EPROBE_DEFER;
>> }
>>
>> parent_port = find_cxl_port(dparent, &parent_dport);
>> diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
>> index 5c7ad230bccb..56fd7a100c2f 100644
>> --- a/drivers/cxl/mem.c
>> +++ b/drivers/cxl/mem.c
>> @@ -145,8 +145,10 @@ static int cxl_mem_probe(struct device *dev)
>> return rc;
>>
>> rc = devm_cxl_enumerate_ports(cxlmd);
>> - if (rc)
>> + if (rc) {
>> + cxlmd->endpoint = ERR_PTR(rc);
>> return rc;
>> + }
>>
>> parent_port = cxl_mem_find_port(cxlmd, &dport);
>> if (!parent_port) {
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index fc0859f841dc..7e4580fb8659 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -57,4 +57,6 @@ int cxl_release_resource(struct cxl_dev_state *cxlds, enum cxl_resource type);
>> void cxl_set_media_ready(struct cxl_dev_state *cxlds);
>> struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
>> struct cxl_dev_state *cxlds);
>> +struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd);
>> +void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
>> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 11/20] cxl: define a driver interface for HPA free space enumaration
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (9 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 10/20] cxl: indicate probe deferral alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-13 17:52 ` Jonathan Cameron
2024-09-07 8:18 ` [PATCH v3 12/20] efx: use acquire_endpoint when looking for free HPA alejandro.lucero-palau
` (8 subsequent siblings)
19 siblings, 1 reply; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
CXL region creation involves allocating capacity from device DPA
(device-physical-address space) and assigning it to decode a given HPA
(host-physical-address space). Before determining how much DPA to
allocate the amount of available HPA must be determined. Also, not all
HPA is create equal, some specifically targets RAM, some target PMEM,
some is prepared for device-memory flows like HDM-D and HDM-DB, and some
is host-only (HDM-H).
Wrap all of those concerns into an API that retrieves a root decoder
(platform CXL window) that fits the specified constraints and the
capacity available for a new region.
Based on https://lore.kernel.org/linux-cxl/168592159290.1948938.13522227102445462976.stgit@dwillia2-xfh.jf.intel.com/
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
Co-developed-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/cxl/core/region.c | 141 ++++++++++++++++++++++++++++++++++++++
drivers/cxl/cxl.h | 3 +
drivers/cxl/cxlmem.h | 3 +
include/linux/cxl/cxl.h | 8 +++
4 files changed, 155 insertions(+)
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index 21ad5f242875..bb227bf894c4 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -703,6 +703,147 @@ static int free_hpa(struct cxl_region *cxlr)
return 0;
}
+struct cxlrd_max_context {
+ struct device *host_bridge;
+ unsigned long flags;
+ resource_size_t max_hpa;
+ struct cxl_root_decoder *cxlrd;
+};
+
+static int find_max_hpa(struct device *dev, void *data)
+{
+ struct cxlrd_max_context *ctx = data;
+ struct cxl_switch_decoder *cxlsd;
+ struct cxl_root_decoder *cxlrd;
+ struct resource *res, *prev;
+ struct cxl_decoder *cxld;
+ resource_size_t max;
+
+ if (!is_root_decoder(dev))
+ return 0;
+
+ cxlrd = to_cxl_root_decoder(dev);
+ cxld = &cxlrd->cxlsd.cxld;
+ if ((cxld->flags & ctx->flags) != ctx->flags) {
+ dev_dbg(dev, "%s, flags not matching: %08lx vs %08lx\n",
+ __func__, cxld->flags, ctx->flags);
+ return 0;
+ }
+
+ /* An accelerator can not be part of an interleaved HPA range. */
+ if (cxld->interleave_ways != 1) {
+ dev_dbg(dev, "%s, interleave_ways not matching\n", __func__);
+ return 0;
+ }
+
+ cxlsd = &cxlrd->cxlsd;
+
+ guard(rwsem_read)(&cxl_region_rwsem);
+ if (ctx->host_bridge != cxlsd->target[0]->dport_dev) {
+ dev_dbg(dev, "%s, HOST BRIDGE DOES NOT MATCH\n", __func__);
+ return 0;
+ }
+
+ /*
+ * Walk the root decoder resource range relying on cxl_region_rwsem to
+ * preclude sibling arrival/departure and find the largest free space
+ * gap.
+ */
+ lockdep_assert_held_read(&cxl_region_rwsem);
+ max = 0;
+ res = cxlrd->res->child;
+ if (!res)
+ max = resource_size(cxlrd->res);
+ else
+ max = 0;
+
+ for (prev = NULL; res; prev = res, res = res->sibling) {
+ struct resource *next = res->sibling;
+ resource_size_t free = 0;
+
+ if (!prev && res->start > cxlrd->res->start) {
+ free = res->start - cxlrd->res->start;
+ max = max(free, max);
+ }
+ if (prev && res->start > prev->end + 1) {
+ free = res->start - prev->end + 1;
+ max = max(free, max);
+ }
+ if (next && res->end + 1 < next->start) {
+ free = next->start - res->end + 1;
+ max = max(free, max);
+ }
+ if (!next && res->end + 1 < cxlrd->res->end + 1) {
+ free = cxlrd->res->end + 1 - res->end + 1;
+ max = max(free, max);
+ }
+ }
+
+ dev_dbg(CXLRD_DEV(cxlrd), "%s, found %pa bytes of free space\n",
+ __func__, &max);
+ if (max > ctx->max_hpa) {
+ if (ctx->cxlrd)
+ put_device(CXLRD_DEV(ctx->cxlrd));
+ get_device(CXLRD_DEV(cxlrd));
+ ctx->cxlrd = cxlrd;
+ ctx->max_hpa = max;
+ dev_dbg(CXLRD_DEV(cxlrd), "%s, found %pa bytes of free space\n",
+ __func__, &max);
+ }
+ return 0;
+}
+
+/**
+ * cxl_get_hpa_freespace - find a root decoder with free capacity per constraints
+ * @endpoint: an endpoint that is mapped by the returned decoder
+ * @flags: CXL_DECODER_F flags for selecting RAM vs PMEM, and HDM-H vs HDM-D[B]
+ * @max_avail_contig: output parameter of max contiguous bytes available in the
+ * returned decoder
+ *
+ * The return tuple of a 'struct cxl_root_decoder' and 'bytes available (@max)'
+ * is a point in time snapshot. If by the time the caller goes to use this root
+ * decoder's capacity the capacity is reduced then caller needs to loop and
+ * retry.
+ *
+ * The returned root decoder has an elevated reference count that needs to be
+ * put with put_device(cxlrd_dev(cxlrd)). Locking context is with
+ * cxl_{acquire,release}_endpoint(), that ensures removal of the root decoder
+ * does not race.
+ */
+struct cxl_root_decoder *cxl_get_hpa_freespace(struct cxl_port *endpoint,
+ unsigned long flags,
+ resource_size_t *max_avail_contig)
+{
+ struct cxlrd_max_context ctx = {
+ .host_bridge = endpoint->host_bridge,
+ .flags = flags,
+ };
+ struct cxl_port *root_port;
+ struct cxl_root *root __free(put_cxl_root) = find_cxl_root(endpoint);
+
+ if (!is_cxl_endpoint(endpoint)) {
+ dev_dbg(&endpoint->dev, "hpa requestor is not an endpoint\n");
+ return ERR_PTR(-EINVAL);
+ }
+
+ if (!root) {
+ dev_dbg(&endpoint->dev, "endpoint can not be related to a root port\n");
+ return ERR_PTR(-ENXIO);
+ }
+
+ root_port = &root->port;
+ down_read(&cxl_region_rwsem);
+ device_for_each_child(&root_port->dev, &ctx, find_max_hpa);
+ up_read(&cxl_region_rwsem);
+
+ if (!ctx.cxlrd)
+ return ERR_PTR(-ENOMEM);
+
+ *max_avail_contig = ctx.max_hpa;
+ return ctx.cxlrd;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_get_hpa_freespace, CXL);
+
static ssize_t size_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t len)
{
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 07c153aa3d77..5d83e4a960ef 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -772,6 +772,9 @@ static inline void cxl_setup_parent_dport(struct device *host,
struct cxl_decoder *to_cxl_decoder(struct device *dev);
struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev);
struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev);
+
+#define CXLRD_DEV(cxlrd) (&(cxlrd)->cxlsd.cxld.dev)
+
struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev);
bool is_root_decoder(struct device *dev);
bool is_switch_decoder(struct device *dev);
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index 37c043100300..07259840da8f 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -875,4 +875,7 @@ struct cxl_hdm {
struct seq_file;
struct dentry *cxl_debugfs_create_dir(const char *dir);
void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds);
+struct cxl_root_decoder *cxl_get_hpa_freespace(struct cxl_port *endpoint,
+ unsigned long flags,
+ resource_size_t *max);
#endif /* __CXL_MEM_H__ */
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index 7e4580fb8659..60a32f60401f 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -7,6 +7,10 @@
#include <linux/device.h>
#include <linux/pci.h>
+#define CXL_DECODER_F_RAM BIT(0)
+#define CXL_DECODER_F_PMEM BIT(1)
+#define CXL_DECODER_F_TYPE2 BIT(2)
+
enum cxl_resource {
CXL_ACCEL_RES_DPA,
CXL_ACCEL_RES_RAM,
@@ -59,4 +63,8 @@ struct cxl_memdev *devm_cxl_add_memdev(struct device *host,
struct cxl_dev_state *cxlds);
struct cxl_port *cxl_acquire_endpoint(struct cxl_memdev *cxlmd);
void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
+
+struct cxl_root_decoder *cxl_get_hpa_freespace(struct cxl_port *endpoint,
+ unsigned long flags,
+ resource_size_t *max);
#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 11/20] cxl: define a driver interface for HPA free space enumaration
2024-09-07 8:18 ` [PATCH v3 11/20] cxl: define a driver interface for HPA free space enumaration alejandro.lucero-palau
@ 2024-09-13 17:52 ` Jonathan Cameron
2024-09-16 14:09 ` Alejandro Lucero Palau
0 siblings, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 17:52 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:27 +0100
alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> CXL region creation involves allocating capacity from device DPA
> (device-physical-address space) and assigning it to decode a given HPA
> (host-physical-address space). Before determining how much DPA to
> allocate the amount of available HPA must be determined. Also, not all
> HPA is create equal, some specifically targets RAM, some target PMEM,
> some is prepared for device-memory flows like HDM-D and HDM-DB, and some
> is host-only (HDM-H).
>
> Wrap all of those concerns into an API that retrieves a root decoder
> (platform CXL window) that fits the specified constraints and the
> capacity available for a new region.
>
> Based on https://lore.kernel.org/linux-cxl/168592159290.1948938.13522227102445462976.stgit@dwillia2-xfh.jf.intel.com/
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
Trivial comment inline.
J
> ---
> drivers/cxl/core/region.c | 141 ++++++++++++++++++++++++++++++++++++++
> drivers/cxl/cxl.h | 3 +
> drivers/cxl/cxlmem.h | 3 +
> include/linux/cxl/cxl.h | 8 +++
> 4 files changed, 155 insertions(+)
>
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index 21ad5f242875..bb227bf894c4 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
> @@ -703,6 +703,147 @@ static int free_hpa(struct cxl_region *cxlr)
> return 0;
> }
>
> +struct cxlrd_max_context {
> + struct device *host_bridge;
> + unsigned long flags;
> + resource_size_t max_hpa;
> + struct cxl_root_decoder *cxlrd;
> +};
> +
> +static int find_max_hpa(struct device *dev, void *data)
> +{
> + struct cxlrd_max_context *ctx = data;
> + struct cxl_switch_decoder *cxlsd;
> + struct cxl_root_decoder *cxlrd;
> + struct resource *res, *prev;
> + struct cxl_decoder *cxld;
> + resource_size_t max;
> +
> + if (!is_root_decoder(dev))
> + return 0;
> +
> + cxlrd = to_cxl_root_decoder(dev);
> + cxld = &cxlrd->cxlsd.cxld;
> + if ((cxld->flags & ctx->flags) != ctx->flags) {
> + dev_dbg(dev, "%s, flags not matching: %08lx vs %08lx\n",
> + __func__, cxld->flags, ctx->flags);
> + return 0;
> + }
> +
> + /* An accelerator can not be part of an interleaved HPA range. */
> + if (cxld->interleave_ways != 1) {
> + dev_dbg(dev, "%s, interleave_ways not matching\n", __func__);
> + return 0;
> + }
> +
> + cxlsd = &cxlrd->cxlsd;
Perhaps move this before the
cxld = and use it there as well?
> +
> + guard(rwsem_read)(&cxl_region_rwsem);
> + if (ctx->host_bridge != cxlsd->target[0]->dport_dev) {
> + dev_dbg(dev, "%s, HOST BRIDGE DOES NOT MATCH\n", __func__);
Capitals seem a bit ott.
> + return 0;
> + }
> +
> + /*
> + * Walk the root decoder resource range relying on cxl_region_rwsem to
> + * preclude sibling arrival/departure and find the largest free space
> + * gap.
> + */
> + lockdep_assert_held_read(&cxl_region_rwsem);
> + max = 0;
> + res = cxlrd->res->child;
> + if (!res)
> + max = resource_size(cxlrd->res);
> + else
> + max = 0;
> +
> + for (prev = NULL; res; prev = res, res = res->sibling) {
> + struct resource *next = res->sibling;
> + resource_size_t free = 0;
> +
> + if (!prev && res->start > cxlrd->res->start) {
> + free = res->start - cxlrd->res->start;
> + max = max(free, max);
> + }
> + if (prev && res->start > prev->end + 1) {
> + free = res->start - prev->end + 1;
> + max = max(free, max);
> + }
> + if (next && res->end + 1 < next->start) {
> + free = next->start - res->end + 1;
> + max = max(free, max);
> + }
> + if (!next && res->end + 1 < cxlrd->res->end + 1) {
> + free = cxlrd->res->end + 1 - res->end + 1;
> + max = max(free, max);
> + }
> + }
> +
> + dev_dbg(CXLRD_DEV(cxlrd), "%s, found %pa bytes of free space\n",
> + __func__, &max);
> + if (max > ctx->max_hpa) {
> + if (ctx->cxlrd)
> + put_device(CXLRD_DEV(ctx->cxlrd));
> + get_device(CXLRD_DEV(cxlrd));
> + ctx->cxlrd = cxlrd;
> + ctx->max_hpa = max;
> + dev_dbg(CXLRD_DEV(cxlrd), "%s, found %pa bytes of free space\n",
> + __func__, &max);
> + }
> + return 0;
> +}
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 11/20] cxl: define a driver interface for HPA free space enumaration
2024-09-13 17:52 ` Jonathan Cameron
@ 2024-09-16 14:09 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 14:09 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/13/24 18:52, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:27 +0100
> alejandro.lucero-palau@amd.com wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> CXL region creation involves allocating capacity from device DPA
>> (device-physical-address space) and assigning it to decode a given HPA
>> (host-physical-address space). Before determining how much DPA to
>> allocate the amount of available HPA must be determined. Also, not all
>> HPA is create equal, some specifically targets RAM, some target PMEM,
>> some is prepared for device-memory flows like HDM-D and HDM-DB, and some
>> is host-only (HDM-H).
>>
>> Wrap all of those concerns into an API that retrieves a root decoder
>> (platform CXL window) that fits the specified constraints and the
>> capacity available for a new region.
>>
>> Based on https://lore.kernel.org/linux-cxl/168592159290.1948938.13522227102445462976.stgit@dwillia2-xfh.jf.intel.com/
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> Trivial comment inline.
>
> J
>> ---
>> drivers/cxl/core/region.c | 141 ++++++++++++++++++++++++++++++++++++++
>> drivers/cxl/cxl.h | 3 +
>> drivers/cxl/cxlmem.h | 3 +
>> include/linux/cxl/cxl.h | 8 +++
>> 4 files changed, 155 insertions(+)
>>
>> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
>> index 21ad5f242875..bb227bf894c4 100644
>> --- a/drivers/cxl/core/region.c
>> +++ b/drivers/cxl/core/region.c
>> @@ -703,6 +703,147 @@ static int free_hpa(struct cxl_region *cxlr)
>> return 0;
>> }
>>
>> +struct cxlrd_max_context {
>> + struct device *host_bridge;
>> + unsigned long flags;
>> + resource_size_t max_hpa;
>> + struct cxl_root_decoder *cxlrd;
>> +};
>> +
>> +static int find_max_hpa(struct device *dev, void *data)
>> +{
>> + struct cxlrd_max_context *ctx = data;
>> + struct cxl_switch_decoder *cxlsd;
>> + struct cxl_root_decoder *cxlrd;
>> + struct resource *res, *prev;
>> + struct cxl_decoder *cxld;
>> + resource_size_t max;
>> +
>> + if (!is_root_decoder(dev))
>> + return 0;
>> +
>> + cxlrd = to_cxl_root_decoder(dev);
>> + cxld = &cxlrd->cxlsd.cxld;
>> + if ((cxld->flags & ctx->flags) != ctx->flags) {
>> + dev_dbg(dev, "%s, flags not matching: %08lx vs %08lx\n",
>> + __func__, cxld->flags, ctx->flags);
>> + return 0;
>> + }
>> +
>> + /* An accelerator can not be part of an interleaved HPA range. */
>> + if (cxld->interleave_ways != 1) {
>> + dev_dbg(dev, "%s, interleave_ways not matching\n", __func__);
>> + return 0;
>> + }
>> +
>> + cxlsd = &cxlrd->cxlsd;
> Perhaps move this before the
> cxld = and use it there as well?
>
Yes, I'll do.
>> +
>> + guard(rwsem_read)(&cxl_region_rwsem);
>> + if (ctx->host_bridge != cxlsd->target[0]->dport_dev) {
>> + dev_dbg(dev, "%s, HOST BRIDGE DOES NOT MATCH\n", __func__);
> Capitals seem a bit ott.
I agree!
Thanks
>> + return 0;
>> + }
>> +
>> + /*
>> + * Walk the root decoder resource range relying on cxl_region_rwsem to
>> + * preclude sibling arrival/departure and find the largest free space
>> + * gap.
>> + */
>> + lockdep_assert_held_read(&cxl_region_rwsem);
>> + max = 0;
>> + res = cxlrd->res->child;
>> + if (!res)
>> + max = resource_size(cxlrd->res);
>> + else
>> + max = 0;
>> +
>> + for (prev = NULL; res; prev = res, res = res->sibling) {
>> + struct resource *next = res->sibling;
>> + resource_size_t free = 0;
>> +
>> + if (!prev && res->start > cxlrd->res->start) {
>> + free = res->start - cxlrd->res->start;
>> + max = max(free, max);
>> + }
>> + if (prev && res->start > prev->end + 1) {
>> + free = res->start - prev->end + 1;
>> + max = max(free, max);
>> + }
>> + if (next && res->end + 1 < next->start) {
>> + free = next->start - res->end + 1;
>> + max = max(free, max);
>> + }
>> + if (!next && res->end + 1 < cxlrd->res->end + 1) {
>> + free = cxlrd->res->end + 1 - res->end + 1;
>> + max = max(free, max);
>> + }
>> + }
>> +
>> + dev_dbg(CXLRD_DEV(cxlrd), "%s, found %pa bytes of free space\n",
>> + __func__, &max);
>> + if (max > ctx->max_hpa) {
>> + if (ctx->cxlrd)
>> + put_device(CXLRD_DEV(ctx->cxlrd));
>> + get_device(CXLRD_DEV(cxlrd));
>> + ctx->cxlrd = cxlrd;
>> + ctx->max_hpa = max;
>> + dev_dbg(CXLRD_DEV(cxlrd), "%s, found %pa bytes of free space\n",
>> + __func__, &max);
>> + }
>> + return 0;
>> +}
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 12/20] efx: use acquire_endpoint when looking for free HPA
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (10 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 11/20] cxl: define a driver interface for HPA free space enumaration alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-07 19:33 ` kernel test robot
2024-09-12 23:09 ` Dave Jiang
2024-09-07 8:18 ` [PATCH v3 13/20] cxl: define a driver interface for DPA allocation alejandro.lucero-palau
` (7 subsequent siblings)
19 siblings, 2 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Asking for availbale HPA space is the previous step to try to obtain
an HPA range suitable to accel driver purposes.
Add this call to efx cxl initialization and use acquire_endpoint for
avoiding potential races with cxl port creation.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/net/ethernet/sfc/efx.c | 8 +++++++-
drivers/net/ethernet/sfc/efx_cxl.c | 32 ++++++++++++++++++++++++++++++
2 files changed, 39 insertions(+), 1 deletion(-)
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 3a7406aa950c..08a2f527df16 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -1117,10 +1117,16 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
* used for PIO buffers. If there is no CXL support, or initialization
* fails, efx_cxl_pio_initialised wll be false and legacy PIO buffers
* defined at specific PCI BAR regions will be used.
+ *
+ * The only error to handle is -EPROBE_DEFER happening if the root port
+ * is not there yet.
*/
rc = efx_cxl_init(efx);
- if (rc)
+ if (rc) {
+ if (rc == -EPROBE_DEFER)
+ goto fail2;
pci_err(pci_dev, "CXL initialization failed with error %d\n", rc);
+ }
rc = efx_pci_probe_post_io(efx);
if (rc) {
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
index 899bc823a212..826759caa552 100644
--- a/drivers/net/ethernet/sfc/efx_cxl.c
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -23,6 +23,7 @@ int efx_cxl_init(struct efx_nic *efx)
struct pci_dev *pci_dev = efx->pci_dev;
struct efx_cxl *cxl;
struct resource res;
+ resource_size_t max;
u16 dvsec;
int rc;
@@ -90,7 +91,38 @@ int efx_cxl_init(struct efx_nic *efx)
goto err;
}
+ cxl->endpoint = cxl_acquire_endpoint(cxl->cxlmd);
+ if (IS_ERR(cxl->endpoint)) {
+ rc = PTR_ERR(cxl->endpoint);
+ if (rc != -EPROBE_DEFER) {
+ pci_err(pci_dev, "CXL accel acquire endpoint failed");
+ goto err;
+ }
+ }
+
+ cxl->cxlrd = cxl_get_hpa_freespace(cxl->endpoint,
+ CXL_DECODER_F_RAM | CXL_DECODER_F_TYPE2,
+ &max);
+
+ if (IS_ERR(cxl->cxlrd)) {
+ pci_err(pci_dev, "cxl_get_hpa_freespace failed\n");
+ rc = PTR_ERR(cxl->cxlrd);
+ goto err_release;
+ }
+
+ if (max < EFX_CTPIO_BUFFER_SIZE) {
+ pci_err(pci_dev, "%s: no enough free HPA space %llu < %u\n",
+ __func__, max, EFX_CTPIO_BUFFER_SIZE);
+ rc = -ENOSPC;
+ goto err;
+ }
+
+ cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
+
return 0;
+
+err_release:
+ cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
err:
kfree(cxl->cxlds);
kfree(cxl);
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 12/20] efx: use acquire_endpoint when looking for free HPA
2024-09-07 8:18 ` [PATCH v3 12/20] efx: use acquire_endpoint when looking for free HPA alejandro.lucero-palau
@ 2024-09-07 19:33 ` kernel test robot
2024-09-12 23:09 ` Dave Jiang
1 sibling, 0 replies; 88+ messages in thread
From: kernel test robot @ 2024-09-07 19:33 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: oe-kbuild-all, Alejandro Lucero
Hi,
kernel test robot noticed the following build warnings:
[auto build test WARNING on cxl/next]
[also build test WARNING on linus/master v6.11-rc6 next-20240906]
[cannot apply to cxl/pending horms-ipvs/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/alejandro-lucero-palau-amd-com/cxl-add-type2-device-basic-support/20240907-162231
base: https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git next
patch link: https://lore.kernel.org/r/20240907081836.5801-13-alejandro.lucero-palau%40amd.com
patch subject: [PATCH v3 12/20] efx: use acquire_endpoint when looking for free HPA
config: powerpc-ppc6xx_defconfig (https://download.01.org/0day-ci/archive/20240908/202409080317.xjZq1ABN-lkp@intel.com/config)
compiler: powerpc-linux-gcc (GCC) 13.3.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240908/202409080317.xjZq1ABN-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202409080317.xjZq1ABN-lkp@intel.com/
All warnings (new ones prefixed by >>):
In file included from include/linux/device.h:15,
from include/linux/cxl/cxl.h:7,
from drivers/net/ethernet/sfc/efx_cxl.c:12:
drivers/net/ethernet/sfc/efx_cxl.c: In function 'efx_cxl_init':
>> drivers/net/ethernet/sfc/efx_cxl.c:114:34: warning: format '%llu' expects argument of type 'long long unsigned int', but argument 4 has type 'resource_size_t' {aka 'unsigned int'} [-Wformat=]
114 | pci_err(pci_dev, "%s: no enough free HPA space %llu < %u\n",
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/dev_printk.h:110:30: note: in definition of macro 'dev_printk_index_wrap'
110 | _p_func(dev, fmt, ##__VA_ARGS__); \
| ^~~
include/linux/dev_printk.h:154:56: note: in expansion of macro 'dev_fmt'
154 | dev_printk_index_wrap(_dev_err, KERN_ERR, dev, dev_fmt(fmt), ##__VA_ARGS__)
| ^~~~~~~
include/linux/pci.h:2679:41: note: in expansion of macro 'dev_err'
2679 | #define pci_err(pdev, fmt, arg...) dev_err(&(pdev)->dev, fmt, ##arg)
| ^~~~~~~
drivers/net/ethernet/sfc/efx_cxl.c:114:17: note: in expansion of macro 'pci_err'
114 | pci_err(pci_dev, "%s: no enough free HPA space %llu < %u\n",
| ^~~~~~~
drivers/net/ethernet/sfc/efx_cxl.c:114:67: note: format string is defined here
114 | pci_err(pci_dev, "%s: no enough free HPA space %llu < %u\n",
| ~~~^
| |
| long long unsigned int
| %u
vim +114 drivers/net/ethernet/sfc/efx_cxl.c
> 12 #include <linux/cxl/cxl.h>
13 #include <linux/cxl/pci.h>
14 #include <linux/pci.h>
15
16 #include "net_driver.h"
17 #include "efx_cxl.h"
18
19 #define EFX_CTPIO_BUFFER_SIZE (1024 * 1024 * 256)
20
21 int efx_cxl_init(struct efx_nic *efx)
22 {
23 struct pci_dev *pci_dev = efx->pci_dev;
24 struct efx_cxl *cxl;
25 struct resource res;
26 resource_size_t max;
27 u16 dvsec;
28 int rc;
29
30 efx->efx_cxl_pio_initialised = false;
31
32 dvsec = pci_find_dvsec_capability(pci_dev, PCI_VENDOR_ID_CXL,
33 CXL_DVSEC_PCIE_DEVICE);
34
35 if (!dvsec)
36 return 0;
37
38 pci_dbg(pci_dev, "CXL_DVSEC_PCIE_DEVICE capability found\n");
39
40 efx->cxl = kzalloc(sizeof(*cxl), GFP_KERNEL);
41 if (!efx->cxl)
42 return -ENOMEM;
43
44 cxl = efx->cxl;
45
46 cxl->cxlds = cxl_accel_state_create(&pci_dev->dev);
47 if (IS_ERR(cxl->cxlds)) {
48 pci_err(pci_dev, "CXL accel device state failed");
49 kfree(efx->cxl);
50 return -ENOMEM;
51 }
52
53 cxl_set_dvsec(cxl->cxlds, dvsec);
54 cxl_set_serial(cxl->cxlds, pci_dev->dev.id);
55
56 res = DEFINE_RES_MEM(0, EFX_CTPIO_BUFFER_SIZE);
57 if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_DPA)) {
58 pci_err(pci_dev, "cxl_set_resource DPA failed\n");
59 rc = -EINVAL;
60 goto err;
61 }
62
63 res = DEFINE_RES_MEM_NAMED(0, EFX_CTPIO_BUFFER_SIZE, "ram");
64 if (cxl_set_resource(cxl->cxlds, res, CXL_ACCEL_RES_RAM)) {
65 pci_err(pci_dev, "cxl_set_resource RAM failed\n");
66 rc = -EINVAL;
67 goto err;
68 }
69
70 rc = cxl_pci_accel_setup_regs(pci_dev, cxl->cxlds);
71 if (rc) {
72 pci_err(pci_dev, "CXL accel setup regs failed");
73 goto err;
74 }
75
76 rc = cxl_request_resource(cxl->cxlds, CXL_ACCEL_RES_RAM);
77 if (rc) {
78 pci_err(pci_dev, "CXL request resource failed");
79 goto err;
80 }
81
82 /* We do not have the register about media status. Hardware design
83 * implies it is ready.
84 */
85 cxl_set_media_ready(cxl->cxlds);
86
87 cxl->cxlmd = devm_cxl_add_memdev(&pci_dev->dev, cxl->cxlds);
88 if (IS_ERR(cxl->cxlmd)) {
89 pci_err(pci_dev, "CXL accel memdev creation failed");
90 rc = PTR_ERR(cxl->cxlmd);
91 goto err;
92 }
93
94 cxl->endpoint = cxl_acquire_endpoint(cxl->cxlmd);
95 if (IS_ERR(cxl->endpoint)) {
96 rc = PTR_ERR(cxl->endpoint);
97 if (rc != -EPROBE_DEFER) {
98 pci_err(pci_dev, "CXL accel acquire endpoint failed");
99 goto err;
100 }
101 }
102
103 cxl->cxlrd = cxl_get_hpa_freespace(cxl->endpoint,
104 CXL_DECODER_F_RAM | CXL_DECODER_F_TYPE2,
105 &max);
106
107 if (IS_ERR(cxl->cxlrd)) {
108 pci_err(pci_dev, "cxl_get_hpa_freespace failed\n");
109 rc = PTR_ERR(cxl->cxlrd);
110 goto err_release;
111 }
112
113 if (max < EFX_CTPIO_BUFFER_SIZE) {
> 114 pci_err(pci_dev, "%s: no enough free HPA space %llu < %u\n",
115 __func__, max, EFX_CTPIO_BUFFER_SIZE);
116 rc = -ENOSPC;
117 goto err;
118 }
119
120 cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
121
122 return 0;
123
124 err_release:
125 cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
126 err:
127 kfree(cxl->cxlds);
128 kfree(cxl);
129 efx->cxl = NULL;
130
131 return rc;
132 }
133
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 12/20] efx: use acquire_endpoint when looking for free HPA
2024-09-07 8:18 ` [PATCH v3 12/20] efx: use acquire_endpoint when looking for free HPA alejandro.lucero-palau
2024-09-07 19:33 ` kernel test robot
@ 2024-09-12 23:09 ` Dave Jiang
2024-09-16 10:29 ` Alejandro Lucero Palau
1 sibling, 1 reply; 88+ messages in thread
From: Dave Jiang @ 2024-09-12 23:09 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Asking for availbale HPA space is the previous step to try to obtain
> an HPA range suitable to accel driver purposes.
>
> Add this call to efx cxl initialization and use acquire_endpoint for
> avoiding potential races with cxl port creation.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/net/ethernet/sfc/efx.c | 8 +++++++-
> drivers/net/ethernet/sfc/efx_cxl.c | 32 ++++++++++++++++++++++++++++++
> 2 files changed, 39 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
> index 3a7406aa950c..08a2f527df16 100644
> --- a/drivers/net/ethernet/sfc/efx.c
> +++ b/drivers/net/ethernet/sfc/efx.c
> @@ -1117,10 +1117,16 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
> * used for PIO buffers. If there is no CXL support, or initialization
> * fails, efx_cxl_pio_initialised wll be false and legacy PIO buffers
> * defined at specific PCI BAR regions will be used.
> + *
> + * The only error to handle is -EPROBE_DEFER happening if the root port
> + * is not there yet.
> */
> rc = efx_cxl_init(efx);
> - if (rc)
> + if (rc) {
> + if (rc == -EPROBE_DEFER)
> + goto fail2;
> pci_err(pci_dev, "CXL initialization failed with error %d\n", rc);
> + }
>
> rc = efx_pci_probe_post_io(efx);
> if (rc) {
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index 899bc823a212..826759caa552 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -23,6 +23,7 @@ int efx_cxl_init(struct efx_nic *efx)
> struct pci_dev *pci_dev = efx->pci_dev;
> struct efx_cxl *cxl;
> struct resource res;
> + resource_size_t max;
> u16 dvsec;
> int rc;
>
> @@ -90,7 +91,38 @@ int efx_cxl_init(struct efx_nic *efx)
> goto err;
> }
>
> + cxl->endpoint = cxl_acquire_endpoint(cxl->cxlmd);
> + if (IS_ERR(cxl->endpoint)) {
> + rc = PTR_ERR(cxl->endpoint);
> + if (rc != -EPROBE_DEFER) {
> + pci_err(pci_dev, "CXL accel acquire endpoint failed");
> + goto err;
> + }
What happens if (rc == -EPROBE_DEFER)? Here it drops down but you don't have a valid cxl->endpoint when cxl_get_hpa_freespace() is called.
DJ
> + }
> +
> + cxl->cxlrd = cxl_get_hpa_freespace(cxl->endpoint,
> + CXL_DECODER_F_RAM | CXL_DECODER_F_TYPE2,
> + &max);
> +
> + if (IS_ERR(cxl->cxlrd)) {
> + pci_err(pci_dev, "cxl_get_hpa_freespace failed\n");
> + rc = PTR_ERR(cxl->cxlrd);
> + goto err_release;
> + }
> +
> + if (max < EFX_CTPIO_BUFFER_SIZE) {
> + pci_err(pci_dev, "%s: no enough free HPA space %llu < %u\n",
> + __func__, max, EFX_CTPIO_BUFFER_SIZE);
> + rc = -ENOSPC;
> + goto err;
> + }
> +
> + cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
> +
> return 0;
> +
> +err_release:
> + cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
> err:
> kfree(cxl->cxlds);
> kfree(cxl);
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 12/20] efx: use acquire_endpoint when looking for free HPA
2024-09-12 23:09 ` Dave Jiang
@ 2024-09-16 10:29 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 10:29 UTC (permalink / raw)
To: Dave Jiang, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/13/24 00:09, Dave Jiang wrote:
>
> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Asking for availbale HPA space is the previous step to try to obtain
>> an HPA range suitable to accel driver purposes.
>>
>> Add this call to efx cxl initialization and use acquire_endpoint for
>> avoiding potential races with cxl port creation.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/net/ethernet/sfc/efx.c | 8 +++++++-
>> drivers/net/ethernet/sfc/efx_cxl.c | 32 ++++++++++++++++++++++++++++++
>> 2 files changed, 39 insertions(+), 1 deletion(-)
>>
>> diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
>> index 3a7406aa950c..08a2f527df16 100644
>> --- a/drivers/net/ethernet/sfc/efx.c
>> +++ b/drivers/net/ethernet/sfc/efx.c
>> @@ -1117,10 +1117,16 @@ static int efx_pci_probe(struct pci_dev *pci_dev,
>> * used for PIO buffers. If there is no CXL support, or initialization
>> * fails, efx_cxl_pio_initialised wll be false and legacy PIO buffers
>> * defined at specific PCI BAR regions will be used.
>> + *
>> + * The only error to handle is -EPROBE_DEFER happening if the root port
>> + * is not there yet.
>> */
>> rc = efx_cxl_init(efx);
>> - if (rc)
>> + if (rc) {
>> + if (rc == -EPROBE_DEFER)
>> + goto fail2;
>> pci_err(pci_dev, "CXL initialization failed with error %d\n", rc);
>> + }
>>
>> rc = efx_pci_probe_post_io(efx);
>> if (rc) {
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
>> index 899bc823a212..826759caa552 100644
>> --- a/drivers/net/ethernet/sfc/efx_cxl.c
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -23,6 +23,7 @@ int efx_cxl_init(struct efx_nic *efx)
>> struct pci_dev *pci_dev = efx->pci_dev;
>> struct efx_cxl *cxl;
>> struct resource res;
>> + resource_size_t max;
>> u16 dvsec;
>> int rc;
>>
>> @@ -90,7 +91,38 @@ int efx_cxl_init(struct efx_nic *efx)
>> goto err;
>> }
>>
>> + cxl->endpoint = cxl_acquire_endpoint(cxl->cxlmd);
>> + if (IS_ERR(cxl->endpoint)) {
>> + rc = PTR_ERR(cxl->endpoint);
>> + if (rc != -EPROBE_DEFER) {
>> + pci_err(pci_dev, "CXL accel acquire endpoint failed");
>> + goto err;
>> + }
> What happens if (rc == -EPROBE_DEFER)? Here it drops down but you don't have a valid cxl->endpoint when cxl_get_hpa_freespace() is called.
>
> DJ
>
I missed it!
I thought about implementing a test case for this situation what would
have shown the bug ...
FWIW, the caller is checking that specific return.
Thanks!
>> + }
>> +
>> + cxl->cxlrd = cxl_get_hpa_freespace(cxl->endpoint,
>> + CXL_DECODER_F_RAM | CXL_DECODER_F_TYPE2,
>> + &max);
>> +
>> + if (IS_ERR(cxl->cxlrd)) {
>> + pci_err(pci_dev, "cxl_get_hpa_freespace failed\n");
>> + rc = PTR_ERR(cxl->cxlrd);
>> + goto err_release;
>> + }
>> +
>> + if (max < EFX_CTPIO_BUFFER_SIZE) {
>> + pci_err(pci_dev, "%s: no enough free HPA space %llu < %u\n",
>> + __func__, max, EFX_CTPIO_BUFFER_SIZE);
>> + rc = -ENOSPC;
>> + goto err;
>> + }
>> +
>> + cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
>> +
>> return 0;
>> +
>> +err_release:
>> + cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
>> err:
>> kfree(cxl->cxlds);
>> kfree(cxl);
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 13/20] cxl: define a driver interface for DPA allocation
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (11 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 12/20] efx: use acquire_endpoint when looking for free HPA alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-13 17:59 ` Jonathan Cameron
2024-09-07 8:18 ` [PATCH v3 14/20] cxl: make region type based on endpoint type alejandro.lucero-palau
` (6 subsequent siblings)
19 siblings, 1 reply; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Region creation involves finding available DPA (device-physical-address)
capacity to map into HPA (host-physical-address) space. Given the HPA
capacity constraint, define an API, cxl_request_dpa(), that has the
flexibility to map the minimum amount of memory the driver needs to
operate vs the total possible that can be mapped given HPA availability.
Factor out the core of cxl_dpa_alloc, that does free space scanning,
into a cxl_dpa_freespace() helper, and use that to balance the capacity
available to map vs the @min and @max arguments to cxl_request_dpa.
Based on https://lore.kernel.org/linux-cxl/168592158743.1948938.7622563891193802610.stgit@dwillia2-xfh.jf.intel.com/
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
Co-developed-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/cxl/core/hdm.c | 153 +++++++++++++++++++++++++----
drivers/net/ethernet/sfc/efx_cxl.c | 9 ++
include/linux/cxl/cxl.h | 5 +
3 files changed, 147 insertions(+), 20 deletions(-)
diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c
index 953a5f86a43f..1d034ef7bee3 100644
--- a/drivers/cxl/core/hdm.c
+++ b/drivers/cxl/core/hdm.c
@@ -3,6 +3,7 @@
#include <linux/seq_file.h>
#include <linux/device.h>
#include <linux/delay.h>
+#include <linux/cxl/cxl.h>
#include "cxlmem.h"
#include "core.h"
@@ -418,6 +419,7 @@ int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)
up_write(&cxl_dpa_rwsem);
return rc;
}
+EXPORT_SYMBOL_NS_GPL(cxl_dpa_free, CXL);
int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
enum cxl_decoder_mode mode)
@@ -465,31 +467,18 @@ int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,
return rc;
}
-int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
+static resource_size_t cxl_dpa_freespace(struct cxl_endpoint_decoder *cxled,
+ resource_size_t *start_out,
+ resource_size_t *skip_out)
{
struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
resource_size_t free_ram_start, free_pmem_start;
- struct cxl_port *port = cxled_to_port(cxled);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
- struct device *dev = &cxled->cxld.dev;
resource_size_t start, avail, skip;
struct resource *p, *last;
- int rc;
-
- down_write(&cxl_dpa_rwsem);
- if (cxled->cxld.region) {
- dev_dbg(dev, "decoder attached to %s\n",
- dev_name(&cxled->cxld.region->dev));
- rc = -EBUSY;
- goto out;
- }
- if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
- dev_dbg(dev, "decoder enabled\n");
- rc = -EBUSY;
- goto out;
- }
+ lockdep_assert_held(&cxl_dpa_rwsem);
for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)
last = p;
if (last)
@@ -526,14 +515,45 @@ int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
skip_end = start - 1;
skip = skip_end - skip_start + 1;
} else {
- dev_dbg(dev, "mode not set\n");
- rc = -EINVAL;
+ avail = 0;
+ }
+
+ if (!avail)
+ return 0;
+ if (start_out)
+ *start_out = start;
+ if (skip_out)
+ *skip_out = skip;
+ return avail;
+}
+
+int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
+{
+ struct cxl_port *port = cxled_to_port(cxled);
+ struct device *dev = &cxled->cxld.dev;
+ resource_size_t start, avail, skip;
+ int rc;
+
+ down_write(&cxl_dpa_rwsem);
+ if (cxled->cxld.region) {
+ dev_dbg(dev, "EBUSY, decoder attached to %s\n",
+ dev_name(&cxled->cxld.region->dev));
+ rc = -EBUSY;
goto out;
}
+ if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {
+ dev_dbg(dev, "EBUSY, decoder enabled\n");
+ rc = -EBUSY;
+ goto out;
+ }
+
+ avail = cxl_dpa_freespace(cxled, &start, &skip);
+
if (size > avail) {
dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,
- cxl_decoder_mode_name(cxled->mode), &avail);
+ cxled->mode == CXL_DECODER_RAM ? "ram" : "pmem",
+ &avail);
rc = -ENOSPC;
goto out;
}
@@ -548,6 +568,99 @@ int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)
return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);
}
+static int find_free_decoder(struct device *dev, void *data)
+{
+ struct cxl_endpoint_decoder *cxled;
+ struct cxl_port *port;
+
+ if (!is_endpoint_decoder(dev))
+ return 0;
+
+ cxled = to_cxl_endpoint_decoder(dev);
+ port = cxled_to_port(cxled);
+
+ if (cxled->cxld.id != port->hdm_end + 1)
+ return 0;
+
+ return 1;
+}
+
+/**
+ * cxl_request_dpa - search and reserve DPA given input constraints
+ * @endpoint: an endpoint port with available decoders
+ * @is_ram: DPA operation mode (ram vs pmem)
+ * @min: the minimum amount of capacity the call needs
+ * @max: extra capacity to allocate after min is satisfied
+ *
+ * Given that a region needs to allocate from limited HPA capacity it
+ * may be the case that a device has more mappable DPA capacity than
+ * available HPA. So, the expectation is that @min is a driver known
+ * value for how much capacity is needed, and @max is based the limit of
+ * how much HPA space is available for a new region.
+ *
+ * Returns a pinned cxl_decoder with at least @min bytes of capacity
+ * reserved, or an error pointer. The caller is also expected to own the
+ * lifetime of the memdev registration associated with the endpoint to
+ * pin the decoder registered as well.
+ */
+struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_port *endpoint,
+ bool is_ram,
+ resource_size_t min,
+ resource_size_t max)
+{
+ struct cxl_endpoint_decoder *cxled;
+ enum cxl_decoder_mode mode;
+ struct device *cxled_dev;
+ resource_size_t alloc;
+ int rc;
+
+ if (!IS_ALIGNED(min | max, SZ_256M))
+ return ERR_PTR(-EINVAL);
+
+ down_read(&cxl_dpa_rwsem);
+
+ cxled_dev = device_find_child(&endpoint->dev, NULL, find_free_decoder);
+ if (!cxled_dev)
+ cxled = ERR_PTR(-ENXIO);
+ else
+ cxled = to_cxl_endpoint_decoder(cxled_dev);
+
+ up_read(&cxl_dpa_rwsem);
+
+ if (IS_ERR(cxled))
+ return cxled;
+
+ if (is_ram)
+ mode = CXL_DECODER_RAM;
+ else
+ mode = CXL_DECODER_PMEM;
+
+ rc = cxl_dpa_set_mode(cxled, mode);
+ if (rc)
+ goto err;
+
+ down_read(&cxl_dpa_rwsem);
+ alloc = cxl_dpa_freespace(cxled, NULL, NULL);
+ up_read(&cxl_dpa_rwsem);
+
+ if (max)
+ alloc = min(max, alloc);
+ if (alloc < min) {
+ rc = -ENOMEM;
+ goto err;
+ }
+
+ rc = cxl_dpa_alloc(cxled, alloc);
+ if (rc)
+ goto err;
+
+ return cxled;
+err:
+ put_device(cxled_dev);
+ return ERR_PTR(rc);
+}
+EXPORT_SYMBOL_NS_GPL(cxl_request_dpa, CXL);
+
static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)
{
u16 eig;
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
index 826759caa552..57667d753550 100644
--- a/drivers/net/ethernet/sfc/efx_cxl.c
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -117,6 +117,14 @@ int efx_cxl_init(struct efx_nic *efx)
goto err;
}
+ cxl->cxled = cxl_request_dpa(cxl->endpoint, true, EFX_CTPIO_BUFFER_SIZE,
+ EFX_CTPIO_BUFFER_SIZE);
+ if (IS_ERR(cxl->cxled)) {
+ pci_err(pci_dev, "CXL accel request DPA failed");
+ rc = PTR_ERR(cxl->cxlrd);
+ goto err_release;
+ }
+
cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
return 0;
@@ -134,6 +142,7 @@ int efx_cxl_init(struct efx_nic *efx)
void efx_cxl_exit(struct efx_nic *efx)
{
if (efx->cxl) {
+ cxl_dpa_free(efx->cxl->cxled);
cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
kfree(efx->cxl->cxlds);
kfree(efx->cxl);
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index 60a32f60401f..3250342843e4 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -67,4 +67,9 @@ void cxl_release_endpoint(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
struct cxl_root_decoder *cxl_get_hpa_freespace(struct cxl_port *endpoint,
unsigned long flags,
resource_size_t *max);
+struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_port *endpoint,
+ bool is_ram,
+ resource_size_t min,
+ resource_size_t max);
+int cxl_dpa_free(struct cxl_endpoint_decoder *cxled);
#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 13/20] cxl: define a driver interface for DPA allocation
2024-09-07 8:18 ` [PATCH v3 13/20] cxl: define a driver interface for DPA allocation alejandro.lucero-palau
@ 2024-09-13 17:59 ` Jonathan Cameron
2024-09-16 14:26 ` Alejandro Lucero Palau
0 siblings, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 17:59 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:29 +0100
alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Region creation involves finding available DPA (device-physical-address)
> capacity to map into HPA (host-physical-address) space. Given the HPA
> capacity constraint, define an API, cxl_request_dpa(), that has the
> flexibility to map the minimum amount of memory the driver needs to
> operate vs the total possible that can be mapped given HPA availability.
>
> Factor out the core of cxl_dpa_alloc, that does free space scanning,
> into a cxl_dpa_freespace() helper, and use that to balance the capacity
> available to map vs the @min and @max arguments to cxl_request_dpa.
>
> Based on https://lore.kernel.org/linux-cxl/168592158743.1948938.7622563891193802610.stgit@dwillia2-xfh.jf.intel.com/
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
Trivial comment below.
> +
> +/**
> + * cxl_request_dpa - search and reserve DPA given input constraints
> + * @endpoint: an endpoint port with available decoders
> + * @is_ram: DPA operation mode (ram vs pmem)
> + * @min: the minimum amount of capacity the call needs
> + * @max: extra capacity to allocate after min is satisfied
> + *
> + * Given that a region needs to allocate from limited HPA capacity it
> + * may be the case that a device has more mappable DPA capacity than
> + * available HPA. So, the expectation is that @min is a driver known
> + * value for how much capacity is needed, and @max is based the limit of
> + * how much HPA space is available for a new region.
> + *
> + * Returns a pinned cxl_decoder with at least @min bytes of capacity
> + * reserved, or an error pointer. The caller is also expected to own the
> + * lifetime of the memdev registration associated with the endpoint to
> + * pin the decoder registered as well.
> + */
> +struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_port *endpoint,
> + bool is_ram,
> + resource_size_t min,
> + resource_size_t max)
> +{
> + struct cxl_endpoint_decoder *cxled;
> + enum cxl_decoder_mode mode;
> + struct device *cxled_dev;
> + resource_size_t alloc;
> + int rc;
> +
> + if (!IS_ALIGNED(min | max, SZ_256M))
> + return ERR_PTR(-EINVAL);
> +
> + down_read(&cxl_dpa_rwsem);
> +
> + cxled_dev = device_find_child(&endpoint->dev, NULL, find_free_decoder);
> + if (!cxled_dev)
> + cxled = ERR_PTR(-ENXIO);
> + else
> + cxled = to_cxl_endpoint_decoder(cxled_dev);
Does this need to be under the rwsem? If not cleaner to just
check cxled_dev outside the lock and return the error directly.
Also, in theory this could return NULL - in practice not but
checking it for IS_ERR() is perhaps going to lead to a bug
in the distant future.
> +
> + up_read(&cxl_dpa_rwsem);
> +
> + if (IS_ERR(cxled))
> + return cxled;
> +
> + if (is_ram)
> + mode = CXL_DECODER_RAM;
> + else
> + mode = CXL_DECODER_PMEM;
> +
> + rc = cxl_dpa_set_mode(cxled, mode);
> + if (rc)
> + goto err;
> +
> + down_read(&cxl_dpa_rwsem);
> + alloc = cxl_dpa_freespace(cxled, NULL, NULL);
> + up_read(&cxl_dpa_rwsem);
> +
> + if (max)
> + alloc = min(max, alloc);
> + if (alloc < min) {
> + rc = -ENOMEM;
> + goto err;
> + }
> +
> + rc = cxl_dpa_alloc(cxled, alloc);
> + if (rc)
> + goto err;
> +
> + return cxled;
> +err:
> + put_device(cxled_dev);
> + return ERR_PTR(rc);
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_request_dpa, CXL);
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 13/20] cxl: define a driver interface for DPA allocation
2024-09-13 17:59 ` Jonathan Cameron
@ 2024-09-16 14:26 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 14:26 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/13/24 18:59, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:29 +0100
> alejandro.lucero-palau@amd.com wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Region creation involves finding available DPA (device-physical-address)
>> capacity to map into HPA (host-physical-address) space. Given the HPA
>> capacity constraint, define an API, cxl_request_dpa(), that has the
>> flexibility to map the minimum amount of memory the driver needs to
>> operate vs the total possible that can be mapped given HPA availability.
>>
>> Factor out the core of cxl_dpa_alloc, that does free space scanning,
>> into a cxl_dpa_freespace() helper, and use that to balance the capacity
>> available to map vs the @min and @max arguments to cxl_request_dpa.
>>
>> Based on https://lore.kernel.org/linux-cxl/168592158743.1948938.7622563891193802610.stgit@dwillia2-xfh.jf.intel.com/
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> Co-developed-by: Dan Williams <dan.j.williams@intel.com>
> Trivial comment below.
>
>
>> +
>> +/**
>> + * cxl_request_dpa - search and reserve DPA given input constraints
>> + * @endpoint: an endpoint port with available decoders
>> + * @is_ram: DPA operation mode (ram vs pmem)
>> + * @min: the minimum amount of capacity the call needs
>> + * @max: extra capacity to allocate after min is satisfied
>> + *
>> + * Given that a region needs to allocate from limited HPA capacity it
>> + * may be the case that a device has more mappable DPA capacity than
>> + * available HPA. So, the expectation is that @min is a driver known
>> + * value for how much capacity is needed, and @max is based the limit of
>> + * how much HPA space is available for a new region.
>> + *
>> + * Returns a pinned cxl_decoder with at least @min bytes of capacity
>> + * reserved, or an error pointer. The caller is also expected to own the
>> + * lifetime of the memdev registration associated with the endpoint to
>> + * pin the decoder registered as well.
>> + */
>> +struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_port *endpoint,
>> + bool is_ram,
>> + resource_size_t min,
>> + resource_size_t max)
>> +{
>> + struct cxl_endpoint_decoder *cxled;
>> + enum cxl_decoder_mode mode;
>> + struct device *cxled_dev;
>> + resource_size_t alloc;
>> + int rc;
>> +
>> + if (!IS_ALIGNED(min | max, SZ_256M))
>> + return ERR_PTR(-EINVAL);
>> +
>> + down_read(&cxl_dpa_rwsem);
>> +
>> + cxled_dev = device_find_child(&endpoint->dev, NULL, find_free_decoder);
>> + if (!cxled_dev)
>> + cxled = ERR_PTR(-ENXIO);
>> + else
>> + cxled = to_cxl_endpoint_decoder(cxled_dev);
> Does this need to be under the rwsem? If not cleaner to just
> check cxled_dev outside the lock and return the error directly.
We got a get_device inside device_find_child, so it should be safe to
use to_cxl_endpoint_decoder without the sem.
I'll follow your suggestion.
> Also, in theory this could return NULL - in practice not but
> checking it for IS_ERR() is perhaps going to lead to a bug
> in the distant future.
>
Right. I'll fix it.
Thanks
>> +
>> + up_read(&cxl_dpa_rwsem);
>> +
>> + if (IS_ERR(cxled))
>> + return cxled;
>> +
>> + if (is_ram)
>> + mode = CXL_DECODER_RAM;
>> + else
>> + mode = CXL_DECODER_PMEM;
>> +
>> + rc = cxl_dpa_set_mode(cxled, mode);
>> + if (rc)
>> + goto err;
>> +
>> + down_read(&cxl_dpa_rwsem);
>> + alloc = cxl_dpa_freespace(cxled, NULL, NULL);
>> + up_read(&cxl_dpa_rwsem);
>> +
>> + if (max)
>> + alloc = min(max, alloc);
>> + if (alloc < min) {
>> + rc = -ENOMEM;
>> + goto err;
>> + }
>> +
>> + rc = cxl_dpa_alloc(cxled, alloc);
>> + if (rc)
>> + goto err;
>> +
>> + return cxled;
>> +err:
>> + put_device(cxled_dev);
>> + return ERR_PTR(rc);
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_request_dpa, CXL);
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 14/20] cxl: make region type based on endpoint type
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (12 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 13/20] cxl: define a driver interface for DPA allocation alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-07 8:18 ` [PATCH v3 15/20] cxl/region: factor out interleave ways setup alejandro.lucero-palau
` (5 subsequent siblings)
19 siblings, 0 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Current code is expecting Type3 or CXL_DECODER_HOSTONLYMEM devices only.
Support for Type2 implies region type needs to be based on the endpoint
type instead.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/region.c | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index bb227bf894c4..b27303b9764c 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -2664,7 +2664,8 @@ static ssize_t create_ram_region_show(struct device *dev,
}
static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
- enum cxl_decoder_mode mode, int id)
+ enum cxl_decoder_mode mode, int id,
+ enum cxl_decoder_type target_type)
{
int rc;
@@ -2686,7 +2687,7 @@ static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
return ERR_PTR(-EBUSY);
}
- return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
+ return devm_cxl_add_region(cxlrd, id, mode, target_type);
}
static ssize_t create_pmem_region_store(struct device *dev,
@@ -2701,7 +2702,8 @@ static ssize_t create_pmem_region_store(struct device *dev,
if (rc != 1)
return -EINVAL;
- cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
+ cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id,
+ CXL_DECODER_HOSTONLYMEM);
if (IS_ERR(cxlr))
return PTR_ERR(cxlr);
@@ -2721,7 +2723,8 @@ static ssize_t create_ram_region_store(struct device *dev,
if (rc != 1)
return -EINVAL;
- cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
+ cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id,
+ CXL_DECODER_HOSTONLYMEM);
if (IS_ERR(cxlr))
return PTR_ERR(cxlr);
@@ -3381,7 +3384,8 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
do {
cxlr = __create_region(cxlrd, cxled->mode,
- atomic_read(&cxlrd->region_id));
+ atomic_read(&cxlrd->region_id),
+ cxled->cxld.target_type);
} while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
if (IS_ERR(cxlr)) {
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* [PATCH v3 15/20] cxl/region: factor out interleave ways setup
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (13 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 14/20] cxl: make region type based on endpoint type alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-07 8:18 ` [PATCH v3 16/20] cxl/region: factor out interleave granularity setup alejandro.lucero-palau
` (4 subsequent siblings)
19 siblings, 0 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
In preparation for kernel driven region creation, factor out a common
helper from the user-sysfs region setup for interleave ways.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
Co-developed-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/cxl/core/region.c | 46 +++++++++++++++++++++++----------------
1 file changed, 27 insertions(+), 19 deletions(-)
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index b27303b9764c..edd710105302 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -480,22 +480,14 @@ static ssize_t interleave_ways_show(struct device *dev,
static const struct attribute_group *get_cxl_region_target_group(void);
-static ssize_t interleave_ways_store(struct device *dev,
- struct device_attribute *attr,
- const char *buf, size_t len)
+static int set_interleave_ways(struct cxl_region *cxlr, int val)
{
- struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
- struct cxl_region *cxlr = to_cxl_region(dev);
struct cxl_region_params *p = &cxlr->params;
- unsigned int val, save;
- int rc;
+ int save, rc;
u8 iw;
- rc = kstrtouint(buf, 0, &val);
- if (rc)
- return rc;
-
rc = ways_to_eiw(val, &iw);
if (rc)
return rc;
@@ -510,20 +502,36 @@ static ssize_t interleave_ways_store(struct device *dev,
return -EINVAL;
}
- rc = down_write_killable(&cxl_region_rwsem);
- if (rc)
- return rc;
- if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
- rc = -EBUSY;
- goto out;
- }
+ lockdep_assert_held_write(&cxl_region_rwsem);
+ if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
+ return -EBUSY;
save = p->interleave_ways;
p->interleave_ways = val;
rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
if (rc)
p->interleave_ways = save;
-out:
+
+ return rc;
+}
+
+static ssize_t interleave_ways_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t len)
+{
+ struct cxl_region *cxlr = to_cxl_region(dev);
+ unsigned int val;
+ int rc;
+
+ rc = kstrtouint(buf, 0, &val);
+ if (rc)
+ return rc;
+
+ rc = down_write_killable(&cxl_region_rwsem);
+ if (rc)
+ return rc;
+
+ rc = set_interleave_ways(cxlr, val);
up_write(&cxl_region_rwsem);
if (rc)
return rc;
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* [PATCH v3 16/20] cxl/region: factor out interleave granularity setup
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (14 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 15/20] cxl/region: factor out interleave ways setup alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-07 8:18 ` [PATCH v3 17/20] cxl: allow region creation by type2 drivers alejandro.lucero-palau
` (3 subsequent siblings)
19 siblings, 0 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
In preparation for kernel driven region creation, factor out a common
helper from the user-sysfs region setup for interleave granularity.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
Co-developed-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/cxl/core/region.c | 39 +++++++++++++++++++++++----------------
1 file changed, 23 insertions(+), 16 deletions(-)
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index edd710105302..c6fa9e7e4909 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -556,21 +556,14 @@ static ssize_t interleave_granularity_show(struct device *dev,
return rc;
}
-static ssize_t interleave_granularity_store(struct device *dev,
- struct device_attribute *attr,
- const char *buf, size_t len)
+static int set_interleave_granularity(struct cxl_region *cxlr, int val)
{
- struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
- struct cxl_region *cxlr = to_cxl_region(dev);
struct cxl_region_params *p = &cxlr->params;
- int rc, val;
+ int rc;
u16 ig;
- rc = kstrtoint(buf, 0, &val);
- if (rc)
- return rc;
-
rc = granularity_to_eig(val, &ig);
if (rc)
return rc;
@@ -586,16 +579,30 @@ static ssize_t interleave_granularity_store(struct device *dev,
if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
return -EINVAL;
+ lockdep_assert_held_write(&cxl_region_rwsem);
+ if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
+ return -EBUSY;
+
+ p->interleave_granularity = val;
+ return 0;
+}
+
+static ssize_t interleave_granularity_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t len)
+{
+ struct cxl_region *cxlr = to_cxl_region(dev);
+ int rc, val;
+
+ rc = kstrtoint(buf, 0, &val);
+ if (rc)
+ return rc;
+
rc = down_write_killable(&cxl_region_rwsem);
if (rc)
return rc;
- if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
- rc = -EBUSY;
- goto out;
- }
- p->interleave_granularity = val;
-out:
+ rc = set_interleave_granularity(cxlr, val);
up_write(&cxl_region_rwsem);
if (rc)
return rc;
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* [PATCH v3 17/20] cxl: allow region creation by type2 drivers
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (15 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 16/20] cxl/region: factor out interleave granularity setup alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-13 18:08 ` Jonathan Cameron
2024-09-07 8:18 ` [PATCH v3 18/20] cxl: preclude device memory to be used for dax alejandro.lucero-palau
` (2 subsequent siblings)
19 siblings, 1 reply; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
Creating a CXL region requires userspace intervention through the cxl
sysfs files. Type2 support should allow accelerator drivers to create
such cxl region from kernel code.
Adding that functionality and integrating it with current support for
memory expanders.
Based on https://lore.kernel.org/linux-cxl/168592159835.1948938.1647215579839222774.stgit@dwillia2-xfh.jf.intel.com/
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
---
drivers/cxl/core/region.c | 158 +++++++++++++++++++++++++----
drivers/cxl/cxl.h | 1 +
drivers/cxl/cxlmem.h | 2 +
drivers/net/ethernet/sfc/efx_cxl.c | 10 ++
include/linux/cxl/cxl.h | 4 +
5 files changed, 153 insertions(+), 22 deletions(-)
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index c6fa9e7e4909..d8c29e28e60c 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -2192,7 +2192,7 @@ static int cxl_region_attach(struct cxl_region *cxlr,
return 0;
}
-static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
+int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
{
struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
struct cxl_region *cxlr = cxled->cxld.region;
@@ -2251,6 +2251,7 @@ static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
put_device(&cxlr->dev);
return rc;
}
+EXPORT_SYMBOL_NS_GPL(cxl_region_detach, CXL);
void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
{
@@ -2780,6 +2781,14 @@ cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
return to_cxl_region(region_dev);
}
+static void drop_region(struct cxl_region *cxlr)
+{
+ struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
+ struct cxl_port *port = cxlrd_to_port(cxlrd);
+
+ devm_release_action(port->uport_dev, unregister_region, cxlr);
+}
+
static ssize_t delete_region_store(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t len)
@@ -3385,17 +3394,18 @@ static int match_region_by_range(struct device *dev, void *data)
return rc;
}
-/* Establish an empty region covering the given HPA range */
-static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
- struct cxl_endpoint_decoder *cxled)
+static void construct_region_end(void)
+{
+ up_write(&cxl_region_rwsem);
+}
+
+static struct cxl_region *construct_region_begin(struct cxl_root_decoder *cxlrd,
+ struct cxl_endpoint_decoder *cxled)
{
struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
- struct cxl_port *port = cxlrd_to_port(cxlrd);
- struct range *hpa = &cxled->cxld.hpa_range;
struct cxl_region_params *p;
struct cxl_region *cxlr;
- struct resource *res;
- int rc;
+ int err = 0;
do {
cxlr = __create_region(cxlrd, cxled->mode,
@@ -3404,8 +3414,7 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
} while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
if (IS_ERR(cxlr)) {
- dev_err(cxlmd->dev.parent,
- "%s:%s: %s failed assign region: %ld\n",
+ dev_err(cxlmd->dev.parent, "%s:%s: %s failed assign region: %ld\n",
dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
__func__, PTR_ERR(cxlr));
return cxlr;
@@ -3415,19 +3424,41 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
p = &cxlr->params;
if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
dev_err(cxlmd->dev.parent,
- "%s:%s: %s autodiscovery interrupted\n",
+ "%s:%s: %s region setup interrupted\n",
dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
__func__);
- rc = -EBUSY;
- goto err;
+ err = -EBUSY;
}
+ if (err) {
+ construct_region_end();
+ drop_region(cxlr);
+ return ERR_PTR(err);
+ }
+ return cxlr;
+}
+
+/* Establish an empty region covering the given HPA range */
+static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
+ struct cxl_endpoint_decoder *cxled)
+{
+ struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
+ struct range *hpa = &cxled->cxld.hpa_range;
+ struct cxl_region_params *p;
+ struct cxl_region *cxlr;
+ struct resource *res;
+ int rc;
+
+ cxlr = construct_region_begin(cxlrd, cxled);
+ if (IS_ERR(cxlr))
+ return cxlr;
+
set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
res = kmalloc(sizeof(*res), GFP_KERNEL);
if (!res) {
rc = -ENOMEM;
- goto err;
+ goto out;
}
*res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
@@ -3444,6 +3475,7 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
__func__, dev_name(&cxlr->dev));
}
+ p = &cxlr->params;
p->res = res;
p->interleave_ways = cxled->cxld.interleave_ways;
p->interleave_granularity = cxled->cxld.interleave_granularity;
@@ -3451,24 +3483,106 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
if (rc)
- goto err;
+ goto out;
dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
- dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
- dev_name(&cxlr->dev), p->res, p->interleave_ways,
- p->interleave_granularity);
+ dev_name(&cxlmd->dev),
+ dev_name(&cxled->cxld.dev), __func__,
+ dev_name(&cxlr->dev), p->res,
+ p->interleave_ways,
+ p->interleave_granularity);
/* ...to match put_device() in cxl_add_to_region() */
get_device(&cxlr->dev);
up_write(&cxl_region_rwsem);
+out:
+ construct_region_end();
+ if (rc) {
+ drop_region(cxlr);
+ return ERR_PTR(rc);
+ }
return cxlr;
+}
-err:
- up_write(&cxl_region_rwsem);
- devm_release_action(port->uport_dev, unregister_region, cxlr);
- return ERR_PTR(rc);
+static struct cxl_region *
+__construct_new_region(struct cxl_root_decoder *cxlrd,
+ struct cxl_endpoint_decoder *cxled)
+{
+ struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
+ struct cxl_region_params *p;
+ resource_size_t size = 0;
+ struct cxl_region *cxlr;
+ int rc;
+
+ cxlr = construct_region_begin(cxlrd, cxled);
+ if (IS_ERR(cxlr))
+ return cxlr;
+
+ rc = set_interleave_ways(cxlr, 1);
+ if (rc)
+ goto out;
+
+ rc = set_interleave_granularity(cxlr, cxld->interleave_granularity);
+ if (rc)
+ goto out;
+
+ size = resource_size(cxled->dpa_res);
+
+ rc = alloc_hpa(cxlr, size);
+ if (rc)
+ goto out;
+
+ down_read(&cxl_dpa_rwsem);
+ rc = cxl_region_attach(cxlr, cxled, 0);
+ up_read(&cxl_dpa_rwsem);
+
+ if (rc)
+ goto out;
+
+ rc = cxl_region_decode_commit(cxlr);
+ if (rc)
+ goto out;
+
+ p = &cxlr->params;
+ p->state = CXL_CONFIG_COMMIT;
+out:
+ construct_region_end();
+ if (rc) {
+ drop_region(cxlr);
+ return ERR_PTR(rc);
+ }
+ return cxlr;
+}
+
+/**
+ * cxl_create_region - Establish a region given an endpoint decoder
+ * @cxlrd: root decoder to allocate HPA
+ * @cxled: endpoint decoder with reserved DPA capacity
+ *
+ * Returns a fully formed region in the commit state and attached to the
+ * cxl_region driver.
+ */
+struct cxl_region *cxl_create_region(struct cxl_root_decoder *cxlrd,
+ struct cxl_endpoint_decoder *cxled)
+{
+ struct cxl_region *cxlr;
+
+ mutex_lock(&cxlrd->range_lock);
+ cxlr = __construct_new_region(cxlrd, cxled);
+ mutex_unlock(&cxlrd->range_lock);
+
+ if (IS_ERR(cxlr))
+ return cxlr;
+
+ if (device_attach(&cxlr->dev) <= 0) {
+ dev_err(&cxlr->dev, "failed to create region\n");
+ drop_region(cxlr);
+ return ERR_PTR(-ENODEV);
+ }
+ return cxlr;
}
+EXPORT_SYMBOL_NS_GPL(cxl_create_region, CXL);
int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
{
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 5d83e4a960ef..120e961f2e31 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -903,6 +903,7 @@ void cxl_coordinates_combine(struct access_coordinate *out,
bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port);
+int cxl_region_detach(struct cxl_endpoint_decoder *cxled);
/*
* Unit test builds overrides this to __weak, find the 'strong' version
* of these symbols in tools/testing/cxl/.
diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
index 07259840da8f..b0a66b064c73 100644
--- a/drivers/cxl/cxlmem.h
+++ b/drivers/cxl/cxlmem.h
@@ -878,4 +878,6 @@ void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds);
struct cxl_root_decoder *cxl_get_hpa_freespace(struct cxl_port *endpoint,
unsigned long flags,
resource_size_t *max);
+struct cxl_region *cxl_create_region(struct cxl_root_decoder *cxlrd,
+ struct cxl_endpoint_decoder *cxled);
#endif /* __CXL_MEM_H__ */
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
index 57667d753550..dd2dbfb8ba15 100644
--- a/drivers/net/ethernet/sfc/efx_cxl.c
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -125,10 +125,19 @@ int efx_cxl_init(struct efx_nic *efx)
goto err_release;
}
+ cxl->efx_region = cxl_create_region(cxl->cxlrd, cxl->cxled);
+ if (!cxl->efx_region) {
+ pci_err(pci_dev, "CXL accel create region failed");
+ rc = PTR_ERR(cxl->efx_region);
+ goto err_region;
+ }
+
cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
return 0;
+err_region:
+ cxl_dpa_free(efx->cxl->cxled);
err_release:
cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
err:
@@ -142,6 +151,7 @@ int efx_cxl_init(struct efx_nic *efx)
void efx_cxl_exit(struct efx_nic *efx)
{
if (efx->cxl) {
+ cxl_region_detach(efx->cxl->cxled);
cxl_dpa_free(efx->cxl->cxled);
cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
kfree(efx->cxl->cxlds);
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index 3250342843e4..169683d75030 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -72,4 +72,8 @@ struct cxl_endpoint_decoder *cxl_request_dpa(struct cxl_port *endpoint,
resource_size_t min,
resource_size_t max);
int cxl_dpa_free(struct cxl_endpoint_decoder *cxled);
+struct cxl_region *cxl_create_region(struct cxl_root_decoder *cxlrd,
+ struct cxl_endpoint_decoder *cxled);
+
+int cxl_region_detach(struct cxl_endpoint_decoder *cxled);
#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 17/20] cxl: allow region creation by type2 drivers
2024-09-07 8:18 ` [PATCH v3 17/20] cxl: allow region creation by type2 drivers alejandro.lucero-palau
@ 2024-09-13 18:08 ` Jonathan Cameron
2024-09-16 16:31 ` Alejandro Lucero Palau
0 siblings, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 18:08 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:33 +0100
alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> Creating a CXL region requires userspace intervention through the cxl
> sysfs files. Type2 support should allow accelerator drivers to create
> such cxl region from kernel code.
>
> Adding that functionality and integrating it with current support for
> memory expanders.
>
> Based on https://lore.kernel.org/linux-cxl/168592159835.1948938.1647215579839222774.stgit@dwillia2-xfh.jf.intel.com/
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Sign off doesn't make sense given Dan didn't sent the mail. Co-developed missing?
Minor stuff inline,
Jonathan
> +static struct cxl_region *construct_region_begin(struct cxl_root_decoder *cxlrd,
> + struct cxl_endpoint_decoder *cxled)
> {
> struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
> - struct cxl_port *port = cxlrd_to_port(cxlrd);
> - struct range *hpa = &cxled->cxld.hpa_range;
> struct cxl_region_params *p;
> struct cxl_region *cxlr;
> - struct resource *res;
> - int rc;
> + int err = 0;
>
> do {
> cxlr = __create_region(cxlrd, cxled->mode,
> @@ -3404,8 +3414,7 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
> } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
>
> if (IS_ERR(cxlr)) {
> - dev_err(cxlmd->dev.parent,
> - "%s:%s: %s failed assign region: %ld\n",
> + dev_err(cxlmd->dev.parent, "%s:%s: %s failed assign region: %ld\n",
> dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
> __func__, PTR_ERR(cxlr));
> return cxlr;
> @@ -3415,19 +3424,41 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
> p = &cxlr->params;
> if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
> dev_err(cxlmd->dev.parent,
> - "%s:%s: %s autodiscovery interrupted\n",
> + "%s:%s: %s region setup interrupted\n",
> dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
> __func__);
> - rc = -EBUSY;
> - goto err;
> + err = -EBUSY;
> }
>
> + if (err) {
> + construct_region_end();
Here I'd just release the semaphore not call this warpper as it's taken within this
function I think?
> + drop_region(cxlr);
> + return ERR_PTR(err);
> + }
> + return cxlr;
> +}
> +
> *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
> @@ -3444,6 +3475,7 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
> __func__, dev_name(&cxlr->dev));
> }
>
> + p = &cxlr->params;
> p->res = res;
> p->interleave_ways = cxled->cxld.interleave_ways;
> p->interleave_granularity = cxled->cxld.interleave_granularity;
> @@ -3451,24 +3483,106 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
>
> rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
> if (rc)
> - goto err;
> + goto out;
>
> dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
> - dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
> - dev_name(&cxlr->dev), p->res, p->interleave_ways,
> - p->interleave_granularity);
> + dev_name(&cxlmd->dev),
> + dev_name(&cxled->cxld.dev), __func__,
Avoid reformatting unless necessary. Hard to tell what changed
if anything.
> + dev_name(&cxlr->dev), p->res,
> + p->interleave_ways,
> + p->interleave_granularity);
>
> /* ...to match put_device() in cxl_add_to_region() */
> get_device(&cxlr->dev);
> up_write(&cxl_region_rwsem);
>
> +out:
> + construct_region_end();
As below. I'd have separate error handling path.
Same in the other paths.
> + if (rc) {
> + drop_region(cxlr);
> + return ERR_PTR(rc);
> + }
> return cxlr;
> +}
>
> -err:
> - up_write(&cxl_region_rwsem);
> - devm_release_action(port->uport_dev, unregister_region, cxlr);
> - return ERR_PTR(rc);
> +static struct cxl_region *
> +__construct_new_region(struct cxl_root_decoder *cxlrd,
> + struct cxl_endpoint_decoder *cxled)
> +{
> + struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
> + struct cxl_region_params *p;
> + resource_size_t size = 0;
set in all paths that use it.
> + struct cxl_region *cxlr;
> + int rc;
> +
> + cxlr = construct_region_begin(cxlrd, cxled);
> + if (IS_ERR(cxlr))
> + return cxlr;
> +
> + rc = set_interleave_ways(cxlr, 1);
> + if (rc)
> + goto out;
> +
> + rc = set_interleave_granularity(cxlr, cxld->interleave_granularity);
> + if (rc)
> + goto out;
> +
> + size = resource_size(cxled->dpa_res);
> +
only used once
rc = alloc_hpa(cxlr, resource_size(cxled->dpa_res));
> + rc = alloc_hpa(cxlr, size);
> + if (rc)
> + goto out;
> +
> + down_read(&cxl_dpa_rwsem);
> + rc = cxl_region_attach(cxlr, cxled, 0);
> + up_read(&cxl_dpa_rwsem);
> +
> + if (rc)
> + goto out;
> +
> + rc = cxl_region_decode_commit(cxlr);
> + if (rc)
> + goto out;
> +
> + p = &cxlr->params;
> + p->state = CXL_CONFIG_COMMIT;
> +out:
I'd break out a separate error handling path and
just duplicate the next line.
> + construct_region_end();
> + if (rc) {
> + drop_region(cxlr);
> + return ERR_PTR(rc);
> + }
> + return cxlr;
> +}
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 17/20] cxl: allow region creation by type2 drivers
2024-09-13 18:08 ` Jonathan Cameron
@ 2024-09-16 16:31 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 16:31 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/13/24 19:08, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:33 +0100
> alejandro.lucero-palau@amd.com wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> Creating a CXL region requires userspace intervention through the cxl
>> sysfs files. Type2 support should allow accelerator drivers to create
>> such cxl region from kernel code.
>>
>> Adding that functionality and integrating it with current support for
>> memory expanders.
>>
>> Based on https://lore.kernel.org/linux-cxl/168592159835.1948938.1647215579839222774.stgit@dwillia2-xfh.jf.intel.com/
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
> Sign off doesn't make sense given Dan didn't sent the mail. Co-developed missing?
>
>
> Minor stuff inline,
>
> Jonathan
>
>> +static struct cxl_region *construct_region_begin(struct cxl_root_decoder *cxlrd,
>> + struct cxl_endpoint_decoder *cxled)
>> {
>> struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
>> - struct cxl_port *port = cxlrd_to_port(cxlrd);
>> - struct range *hpa = &cxled->cxld.hpa_range;
>> struct cxl_region_params *p;
>> struct cxl_region *cxlr;
>> - struct resource *res;
>> - int rc;
>> + int err = 0;
>>
>> do {
>> cxlr = __create_region(cxlrd, cxled->mode,
>> @@ -3404,8 +3414,7 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
>> } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
>>
>> if (IS_ERR(cxlr)) {
>> - dev_err(cxlmd->dev.parent,
>> - "%s:%s: %s failed assign region: %ld\n",
>> + dev_err(cxlmd->dev.parent, "%s:%s: %s failed assign region: %ld\n",
>> dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
>> __func__, PTR_ERR(cxlr));
>> return cxlr;
>> @@ -3415,19 +3424,41 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
>> p = &cxlr->params;
>> if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
>> dev_err(cxlmd->dev.parent,
>> - "%s:%s: %s autodiscovery interrupted\n",
>> + "%s:%s: %s region setup interrupted\n",
>> dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
>> __func__);
>> - rc = -EBUSY;
>> - goto err;
>> + err = -EBUSY;
>> }
>>
>> + if (err) {
>> + construct_region_end();
> Here I'd just release the semaphore not call this warpper as it's taken within this
> function I think?
No, it is taken inside construct_region_begin. It is hard to know just
looking at the patch.
>> + drop_region(cxlr);
>> + return ERR_PTR(err);
>> + }
>> + return cxlr;
>> +}
>> +
>> *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
>> @@ -3444,6 +3475,7 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
>> __func__, dev_name(&cxlr->dev));
>> }
>>
>> + p = &cxlr->params;
>> p->res = res;
>> p->interleave_ways = cxled->cxld.interleave_ways;
>> p->interleave_granularity = cxled->cxld.interleave_granularity;
>> @@ -3451,24 +3483,106 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
>>
>> rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
>> if (rc)
>> - goto err;
>> + goto out;
>>
>> dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
>> - dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
>> - dev_name(&cxlr->dev), p->res, p->interleave_ways,
>> - p->interleave_granularity);
>> + dev_name(&cxlmd->dev),
>> + dev_name(&cxled->cxld.dev), __func__,
> Avoid reformatting unless necessary. Hard to tell what changed
> if anything.
I'll fix it.
>> + dev_name(&cxlr->dev), p->res,
>> + p->interleave_ways,
>> + p->interleave_granularity);
>>
>> /* ...to match put_device() in cxl_add_to_region() */
>> get_device(&cxlr->dev);
>> up_write(&cxl_region_rwsem);
>>
>> +out:
>> + construct_region_end();
> As below. I'd have separate error handling path.
> Same in the other paths.
>
OK
>> + if (rc) {
>> + drop_region(cxlr);
>> + return ERR_PTR(rc);
>> + }
>> return cxlr;
>> +}
>>
>> -err:
>> - up_write(&cxl_region_rwsem);
>> - devm_release_action(port->uport_dev, unregister_region, cxlr);
>> - return ERR_PTR(rc);
>> +static struct cxl_region *
>> +__construct_new_region(struct cxl_root_decoder *cxlrd,
>> + struct cxl_endpoint_decoder *cxled)
>> +{
>> + struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
>> + struct cxl_region_params *p;
>> + resource_size_t size = 0;
> set in all paths that use it.
>
>> + struct cxl_region *cxlr;
>> + int rc;
>> +
>> + cxlr = construct_region_begin(cxlrd, cxled);
>> + if (IS_ERR(cxlr))
>> + return cxlr;
>> +
>> + rc = set_interleave_ways(cxlr, 1);
>> + if (rc)
>> + goto out;
>> +
>> + rc = set_interleave_granularity(cxlr, cxld->interleave_granularity);
>> + if (rc)
>> + goto out;
>> +
>> + size = resource_size(cxled->dpa_res);
>> +
> only used once
> rc = alloc_hpa(cxlr, resource_size(cxled->dpa_res));
>
Clever.
I'll do it.
>> + rc = alloc_hpa(cxlr, size);
>> + if (rc)
>> + goto out;
>> +
>> + down_read(&cxl_dpa_rwsem);
>> + rc = cxl_region_attach(cxlr, cxled, 0);
>> + up_read(&cxl_dpa_rwsem);
>> +
>> + if (rc)
>> + goto out;
>> +
>> + rc = cxl_region_decode_commit(cxlr);
>> + if (rc)
>> + goto out;
>> +
>> + p = &cxlr->params;
>> + p->state = CXL_CONFIG_COMMIT;
>> +out:
> I'd break out a separate error handling path and
> just duplicate the next line.
I'll do it.
Thanks!
>
>> + construct_region_end();
>> + if (rc) {
>> + drop_region(cxlr);
>> + return ERR_PTR(rc);
>> + }
>> + return cxlr;
>> +}
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 18/20] cxl: preclude device memory to be used for dax
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (16 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 17/20] cxl: allow region creation by type2 drivers alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-13 17:26 ` Dave Jiang
2024-09-07 8:18 ` [PATCH v3 19/20] cxl: add function for obtaining params from a region alejandro.lucero-palau
2024-09-07 8:18 ` [PATCH v3 20/20] efx: support pio mapping based on cxl alejandro.lucero-palau
19 siblings, 1 reply; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
By definition a type2 cxl device will use the host managed memory for
specific functionality, therefore it should not be available to other
uses.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/region.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index d8c29e28e60c..45b4891035a6 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -3699,6 +3699,9 @@ static int cxl_region_probe(struct device *dev)
case CXL_DECODER_PMEM:
return devm_cxl_add_pmem_region(cxlr);
case CXL_DECODER_RAM:
+ if (cxlr->type != CXL_DECODER_HOSTONLYMEM)
+ return 0;
+
/*
* The region can not be manged by CXL if any portion of
* it is already online as 'System RAM'
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 18/20] cxl: preclude device memory to be used for dax
2024-09-07 8:18 ` [PATCH v3 18/20] cxl: preclude device memory to be used for dax alejandro.lucero-palau
@ 2024-09-13 17:26 ` Dave Jiang
2024-09-16 14:32 ` Alejandro Lucero Palau
0 siblings, 1 reply; 88+ messages in thread
From: Dave Jiang @ 2024-09-13 17:26 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> By definition a type2 cxl device will use the host managed memory for
> specific functionality, therefore it should not be available to other
> uses.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/region.c | 3 +++
> 1 file changed, 3 insertions(+)
>
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index d8c29e28e60c..45b4891035a6 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
> @@ -3699,6 +3699,9 @@ static int cxl_region_probe(struct device *dev)
> case CXL_DECODER_PMEM:
> return devm_cxl_add_pmem_region(cxlr);
> case CXL_DECODER_RAM:
> + if (cxlr->type != CXL_DECODER_HOSTONLYMEM)
> + return 0;
> +
While it's highly unlikely that someone puts pmem on a type2 device, the spec does not forbid it. Maybe it makes sense to just return 0 above this switch block entirely and skip this code block?
> /*
> * The region can not be manged by CXL if any portion of
> * it is already online as 'System RAM'
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 18/20] cxl: preclude device memory to be used for dax
2024-09-13 17:26 ` Dave Jiang
@ 2024-09-16 14:32 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 14:32 UTC (permalink / raw)
To: Dave Jiang, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/13/24 18:26, Dave Jiang wrote:
>
> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> By definition a type2 cxl device will use the host managed memory for
>> specific functionality, therefore it should not be available to other
>> uses.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/region.c | 3 +++
>> 1 file changed, 3 insertions(+)
>>
>> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
>> index d8c29e28e60c..45b4891035a6 100644
>> --- a/drivers/cxl/core/region.c
>> +++ b/drivers/cxl/core/region.c
>> @@ -3699,6 +3699,9 @@ static int cxl_region_probe(struct device *dev)
>> case CXL_DECODER_PMEM:
>> return devm_cxl_add_pmem_region(cxlr);
>> case CXL_DECODER_RAM:
>> + if (cxlr->type != CXL_DECODER_HOSTONLYMEM)
>> + return 0;
>> +
> While it's highly unlikely that someone puts pmem on a type2 device, the spec does not forbid it. Maybe it makes sense to just return 0 above this switch block entirely and skip this code block?
Yes, it makes sense.
I'll do it.
Thanks
>
>> /*
>> * The region can not be manged by CXL if any portion of
>> * it is already online as 'System RAM'
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 19/20] cxl: add function for obtaining params from a region
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (17 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 18/20] cxl: preclude device memory to be used for dax alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-13 17:48 ` Dave Jiang
2024-09-07 8:18 ` [PATCH v3 20/20] efx: support pio mapping based on cxl alejandro.lucero-palau
19 siblings, 1 reply; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
A CXL region struct contains the physical address to work with.
Add a function for given a opaque cxl region struct returns the params
to be used for mapping such memory range.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/cxl/core/region.c | 16 ++++++++++++++++
drivers/cxl/cxl.h | 2 ++
include/linux/cxl/cxl.h | 2 ++
3 files changed, 20 insertions(+)
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index 45b4891035a6..e0e2342bb1ed 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -2662,6 +2662,22 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
return ERR_PTR(rc);
}
+int cxl_get_region_params(struct cxl_region *region, resource_size_t *start,
+ resource_size_t *end)
+{
+ if (!region)
+ return -ENODEV;
+
+ if (!region->params.res)
+ return -ENOSPC;
+
+ *start = region->params.res->start;
+ *end = region->params.res->end;
+
+ return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_get_region_params, CXL);
+
static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
{
return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 120e961f2e31..b26833ff52c0 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -904,6 +904,8 @@ void cxl_coordinates_combine(struct access_coordinate *out,
bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port);
int cxl_region_detach(struct cxl_endpoint_decoder *cxled);
+int cxl_get_region_params(struct cxl_region *region, resource_size_t *start,
+ resource_size_t *end);
/*
* Unit test builds overrides this to __weak, find the 'strong' version
* of these symbols in tools/testing/cxl/.
diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
index 169683d75030..ef3bd8329bd8 100644
--- a/include/linux/cxl/cxl.h
+++ b/include/linux/cxl/cxl.h
@@ -76,4 +76,6 @@ struct cxl_region *cxl_create_region(struct cxl_root_decoder *cxlrd,
struct cxl_endpoint_decoder *cxled);
int cxl_region_detach(struct cxl_endpoint_decoder *cxled);
+int cxl_get_region_params(struct cxl_region *region, resource_size_t *start,
+ resource_size_t *end);
#endif
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 19/20] cxl: add function for obtaining params from a region
2024-09-07 8:18 ` [PATCH v3 19/20] cxl: add function for obtaining params from a region alejandro.lucero-palau
@ 2024-09-13 17:48 ` Dave Jiang
2024-09-16 16:22 ` Alejandro Lucero Palau
0 siblings, 1 reply; 88+ messages in thread
From: Dave Jiang @ 2024-09-13 17:48 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> A CXL region struct contains the physical address to work with.
>
> Add a function for given a opaque cxl region struct returns the params
> to be used for mapping such memory range.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/cxl/core/region.c | 16 ++++++++++++++++
> drivers/cxl/cxl.h | 2 ++
> include/linux/cxl/cxl.h | 2 ++
> 3 files changed, 20 insertions(+)
>
> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
> index 45b4891035a6..e0e2342bb1ed 100644
> --- a/drivers/cxl/core/region.c
> +++ b/drivers/cxl/core/region.c
> @@ -2662,6 +2662,22 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
> return ERR_PTR(rc);
> }
>
> +int cxl_get_region_params(struct cxl_region *region, resource_size_t *start,
> + resource_size_t *end)
Maybe just pass in a 'struct range' to be filled out instead of start/end?
> +{
> + if (!region)
> + return -ENODEV;
> +
> + if (!region->params.res)
> + return -ENOSPC;
> +
> + *start = region->params.res->start;
> + *end = region->params.res->end;
> +
> + return 0;
> +}
> +EXPORT_SYMBOL_NS_GPL(cxl_get_region_params, CXL);
I think at least for me, it's better to have the introduction of helper functions to be with the code where it gets called to provide the full picture and thus make a better review. Especially if the function is fairly small. So maybe squash this patch and the next one. There may be a few other situations like this in this series worth the same consideration.
> +
> static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
> {
> return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
> index 120e961f2e31..b26833ff52c0 100644
> --- a/drivers/cxl/cxl.h
> +++ b/drivers/cxl/cxl.h
> @@ -904,6 +904,8 @@ void cxl_coordinates_combine(struct access_coordinate *out,
> bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port);
>
> int cxl_region_detach(struct cxl_endpoint_decoder *cxled);
> +int cxl_get_region_params(struct cxl_region *region, resource_size_t *start,
> + resource_size_t *end);
> /*
> * Unit test builds overrides this to __weak, find the 'strong' version
> * of these symbols in tools/testing/cxl/.
> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
> index 169683d75030..ef3bd8329bd8 100644
> --- a/include/linux/cxl/cxl.h
> +++ b/include/linux/cxl/cxl.h
> @@ -76,4 +76,6 @@ struct cxl_region *cxl_create_region(struct cxl_root_decoder *cxlrd,
> struct cxl_endpoint_decoder *cxled);
>
> int cxl_region_detach(struct cxl_endpoint_decoder *cxled);
> +int cxl_get_region_params(struct cxl_region *region, resource_size_t *start,
> + resource_size_t *end);
> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 19/20] cxl: add function for obtaining params from a region
2024-09-13 17:48 ` Dave Jiang
@ 2024-09-16 16:22 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 16:22 UTC (permalink / raw)
To: Dave Jiang, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/13/24 18:48, Dave Jiang wrote:
>
> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> A CXL region struct contains the physical address to work with.
>>
>> Add a function for given a opaque cxl region struct returns the params
>> to be used for mapping such memory range.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/cxl/core/region.c | 16 ++++++++++++++++
>> drivers/cxl/cxl.h | 2 ++
>> include/linux/cxl/cxl.h | 2 ++
>> 3 files changed, 20 insertions(+)
>>
>> diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
>> index 45b4891035a6..e0e2342bb1ed 100644
>> --- a/drivers/cxl/core/region.c
>> +++ b/drivers/cxl/core/region.c
>> @@ -2662,6 +2662,22 @@ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
>> return ERR_PTR(rc);
>> }
>>
>> +int cxl_get_region_params(struct cxl_region *region, resource_size_t *start,
>> + resource_size_t *end)
> Maybe just pass in a 'struct range' to be filled out instead of start/end?
>
It makes sense.
I'll do it.
>> +{
>> + if (!region)
>> + return -ENODEV;
>> +
>> + if (!region->params.res)
>> + return -ENOSPC;
>> +
>> + *start = region->params.res->start;
>> + *end = region->params.res->end;
>> +
>> + return 0;
>> +}
>> +EXPORT_SYMBOL_NS_GPL(cxl_get_region_params, CXL);
> I think at least for me, it's better to have the introduction of helper functions to be with the code where it gets called to provide the full picture and thus make a better review. Especially if the function is fairly small. So maybe squash this patch and the next one. There may be a few other situations like this in this series worth the same consideration.
>
The next patch is not too big either, but I wanted to make things easier
in that one.
Anyway, I'll do so.
Thanks
>> +
>> static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
>> {
>> return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
>> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
>> index 120e961f2e31..b26833ff52c0 100644
>> --- a/drivers/cxl/cxl.h
>> +++ b/drivers/cxl/cxl.h
>> @@ -904,6 +904,8 @@ void cxl_coordinates_combine(struct access_coordinate *out,
>> bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port);
>>
>> int cxl_region_detach(struct cxl_endpoint_decoder *cxled);
>> +int cxl_get_region_params(struct cxl_region *region, resource_size_t *start,
>> + resource_size_t *end);
>> /*
>> * Unit test builds overrides this to __weak, find the 'strong' version
>> * of these symbols in tools/testing/cxl/.
>> diff --git a/include/linux/cxl/cxl.h b/include/linux/cxl/cxl.h
>> index 169683d75030..ef3bd8329bd8 100644
>> --- a/include/linux/cxl/cxl.h
>> +++ b/include/linux/cxl/cxl.h
>> @@ -76,4 +76,6 @@ struct cxl_region *cxl_create_region(struct cxl_root_decoder *cxlrd,
>> struct cxl_endpoint_decoder *cxled);
>>
>> int cxl_region_detach(struct cxl_endpoint_decoder *cxled);
>> +int cxl_get_region_params(struct cxl_region *region, resource_size_t *start,
>> + resource_size_t *end);
>> #endif
^ permalink raw reply [flat|nested] 88+ messages in thread
* [PATCH v3 20/20] efx: support pio mapping based on cxl
2024-09-07 8:18 [PATCH v3 00/20] cxl: add Type2 device support alejandro.lucero-palau
` (18 preceding siblings ...)
2024-09-07 8:18 ` [PATCH v3 19/20] cxl: add function for obtaining params from a region alejandro.lucero-palau
@ 2024-09-07 8:18 ` alejandro.lucero-palau
2024-09-13 17:45 ` Edward Cree
` (2 more replies)
19 siblings, 3 replies; 88+ messages in thread
From: alejandro.lucero-palau @ 2024-09-07 8:18 UTC (permalink / raw)
To: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
From: Alejandro Lucero <alucerop@amd.com>
With a device supporting CXL and successfully initialised, use the cxl
region to map the memory range and use this mapping for PIO buffers.
Signed-off-by: Alejandro Lucero <alucerop@amd.com>
---
drivers/net/ethernet/sfc/ef10.c | 32 +++++++++++++++++++++------
drivers/net/ethernet/sfc/efx_cxl.c | 20 ++++++++++++++++-
drivers/net/ethernet/sfc/mcdi_pcol.h | 12 ++++++++++
drivers/net/ethernet/sfc/net_driver.h | 2 ++
drivers/net/ethernet/sfc/nic.h | 2 ++
5 files changed, 60 insertions(+), 8 deletions(-)
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index 7d69302ffa0a..d4e64cd0f7a4 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -24,6 +24,7 @@
#include <linux/wait.h>
#include <linux/workqueue.h>
#include <net/udp_tunnel.h>
+#include "efx_cxl.h"
/* Hardware control for EF10 architecture including 'Huntington'. */
@@ -177,6 +178,12 @@ static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
efx->num_mac_stats);
}
+ if (outlen < MC_CMD_GET_CAPABILITIES_V7_OUT_LEN)
+ nic_data->datapath_caps3 = 0;
+ else
+ nic_data->datapath_caps3 = MCDI_DWORD(outbuf,
+ GET_CAPABILITIES_V7_OUT_FLAGS3);
+
return 0;
}
@@ -949,7 +956,7 @@ static void efx_ef10_remove(struct efx_nic *efx)
efx_mcdi_rx_free_indir_table(efx);
- if (nic_data->wc_membase)
+ if (nic_data->wc_membase && !efx->efx_cxl_pio_in_use)
iounmap(nic_data->wc_membase);
rc = efx_mcdi_free_vis(efx);
@@ -1263,8 +1270,19 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
iounmap(efx->membase);
efx->membase = membase;
- /* Set up the WC mapping if needed */
- if (wc_mem_map_size) {
+ if (!wc_mem_map_size)
+ return 0;
+
+ /* Using PIO through CXL mapping? */
+ if ((nic_data->datapath_caps3 &
+ (1 << MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN)) &&
+ efx->efx_cxl_pio_initialised) {
+ nic_data->pio_write_base = efx->cxl->ctpio_cxl +
+ (pio_write_vi_base * efx->vi_stride +
+ ER_DZ_TX_PIOBUF - uc_mem_map_size);
+ efx->efx_cxl_pio_in_use = true;
+ } else {
+ /* Using legacy PIO BAR mapping */
nic_data->wc_membase = ioremap_wc(efx->membase_phys +
uc_mem_map_size,
wc_mem_map_size);
@@ -1279,12 +1297,12 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
nic_data->wc_membase +
(pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF -
uc_mem_map_size);
-
- rc = efx_ef10_link_piobufs(efx);
- if (rc)
- efx_ef10_free_piobufs(efx);
}
+ rc = efx_ef10_link_piobufs(efx);
+ if (rc)
+ efx_ef10_free_piobufs(efx);
+
netif_dbg(efx, probe, efx->net_dev,
"memory BAR at %pa (virtual %p+%x UC, %p+%x WC)\n",
&efx->membase_phys, efx->membase, uc_mem_map_size,
diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
index dd2dbfb8ba15..ef57f833b8a7 100644
--- a/drivers/net/ethernet/sfc/efx_cxl.c
+++ b/drivers/net/ethernet/sfc/efx_cxl.c
@@ -21,9 +21,9 @@
int efx_cxl_init(struct efx_nic *efx)
{
struct pci_dev *pci_dev = efx->pci_dev;
+ resource_size_t start, end, max = 0;
struct efx_cxl *cxl;
struct resource res;
- resource_size_t max;
u16 dvsec;
int rc;
@@ -132,10 +132,27 @@ int efx_cxl_init(struct efx_nic *efx)
goto err_region;
}
+ rc = cxl_get_region_params(cxl->efx_region, &start, &end);
+ if (rc) {
+ pci_err(pci_dev, "CXL getting regions params failed");
+ goto err_map;
+ }
+
+ cxl->ctpio_cxl = ioremap(start, end - start);
+ if (!cxl->ctpio_cxl) {
+ pci_err(pci_dev, "CXL ioremap region failed");
+ rc = -EIO;
+ goto err_map;
+ }
+
+ efx->efx_cxl_pio_initialised = true;
+
cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
return 0;
+err_map:
+ cxl_region_detach(cxl->cxled);
err_region:
cxl_dpa_free(efx->cxl->cxled);
err_release:
@@ -151,6 +168,7 @@ int efx_cxl_init(struct efx_nic *efx)
void efx_cxl_exit(struct efx_nic *efx)
{
if (efx->cxl) {
+ iounmap(efx->cxl->ctpio_cxl);
cxl_region_detach(efx->cxl->cxled);
cxl_dpa_free(efx->cxl->cxled);
cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
index cd297e19cddc..c158a1e8d01b 100644
--- a/drivers/net/ethernet/sfc/mcdi_pcol.h
+++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
@@ -16799,6 +16799,9 @@
#define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
#define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
#define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
+#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_OFST 148
+#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN 17
+#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_WIDTH 1
/* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */
#define MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160
@@ -17303,6 +17306,9 @@
#define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
#define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
#define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
+#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_OFST 148
+#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_LBN 17
+#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_WIDTH 1
/* These bits are reserved for communicating test-specific capabilities to
* host-side test software. All production drivers should treat this field as
* opaque.
@@ -17821,6 +17827,9 @@
#define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
#define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
#define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
+#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_OFST 148
+#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_LBN 17
+#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_WIDTH 1
/* These bits are reserved for communicating test-specific capabilities to
* host-side test software. All production drivers should treat this field as
* opaque.
@@ -18374,6 +18383,9 @@
#define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
#define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
#define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
+#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_OFST 148
+#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_LBN 17
+#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_WIDTH 1
/* These bits are reserved for communicating test-specific capabilities to
* host-side test software. All production drivers should treat this field as
* opaque.
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index 77261de65e63..893e7841ffb4 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -967,6 +967,7 @@ struct efx_cxl;
* @dl_port: devlink port associated with the PF
* @cxl: details of related cxl objects
* @efx_cxl_pio_initialised: clx initialization outcome.
+ * @efx_cxl_pio_in_use: PIO using CXL mapping
* @mem_bar: The BAR that is mapped into membase.
* @reg_base: Offset from the start of the bar to the function control window.
* @monitor_work: Hardware monitor workitem
@@ -1154,6 +1155,7 @@ struct efx_nic {
struct devlink_port *dl_port;
struct efx_cxl *cxl;
bool efx_cxl_pio_initialised;
+ bool efx_cxl_pio_in_use;
unsigned int mem_bar;
u32 reg_base;
diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
index 1db64fc6e909..b7148810acdb 100644
--- a/drivers/net/ethernet/sfc/nic.h
+++ b/drivers/net/ethernet/sfc/nic.h
@@ -151,6 +151,7 @@ enum {
* @datapath_caps: Capabilities of datapath firmware (FLAGS1 field of
* %MC_CMD_GET_CAPABILITIES response)
* @datapath_caps2: Further Capabilities of datapath firmware (FLAGS2 field of
+ * @datapath_caps3: Further Capabilities of datapath firmware (FLAGS3 field of
* %MC_CMD_GET_CAPABILITIES response)
* @rx_dpcpu_fw_id: Firmware ID of the RxDPCPU
* @tx_dpcpu_fw_id: Firmware ID of the TxDPCPU
@@ -186,6 +187,7 @@ struct efx_ef10_nic_data {
bool must_check_datapath_caps;
u32 datapath_caps;
u32 datapath_caps2;
+ u32 datapath_caps3;
unsigned int rx_dpcpu_fw_id;
unsigned int tx_dpcpu_fw_id;
bool must_probe_vswitching;
--
2.17.1
^ permalink raw reply related [flat|nested] 88+ messages in thread* Re: [PATCH v3 20/20] efx: support pio mapping based on cxl
2024-09-07 8:18 ` [PATCH v3 20/20] efx: support pio mapping based on cxl alejandro.lucero-palau
@ 2024-09-13 17:45 ` Edward Cree
2024-09-16 16:12 ` Alejandro Lucero Palau
2024-09-13 17:52 ` Dave Jiang
2024-09-13 18:10 ` Jonathan Cameron
2 siblings, 1 reply; 88+ messages in thread
From: Edward Cree @ 2024-09-13 17:45 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 07/09/2024 09:18, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> With a device supporting CXL and successfully initialised, use the cxl
> region to map the memory range and use this mapping for PIO buffers.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
<snip>> diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
> index cd297e19cddc..c158a1e8d01b 100644
> --- a/drivers/net/ethernet/sfc/mcdi_pcol.h
> +++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
> @@ -16799,6 +16799,9 @@
> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>
> /* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */
> #define MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160
> @@ -17303,6 +17306,9 @@
> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_WIDTH 1
> /* These bits are reserved for communicating test-specific capabilities to
> * host-side test software. All production drivers should treat this field as
> * opaque.
> @@ -17821,6 +17827,9 @@
> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_WIDTH 1
> /* These bits are reserved for communicating test-specific capabilities to
> * host-side test software. All production drivers should treat this field as
> * opaque.
> @@ -18374,6 +18383,9 @@
> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_WIDTH 1
> /* These bits are reserved for communicating test-specific capabilities to
> * host-side test software. All production drivers should treat this field as
> * opaque.
Please, do not make targeted edits to mcdi_pcol.h. Our standard
process for this file is to regenerate the whole thing from
smartnic_registry whenever we want to pick up new additions;
this helps us have certainty and traceability that the driver
and firmware-side definitions of the MCDI protocol are consistent
and limit the risk of copy/paste errors etc. introducing
differences that could cause backwards-compatibility headaches
later. Ideally the MCDI update should also be a separate commit.
See previous commits to this file, such as
1f17708b47a9 ("sfc: update MCDI protocol headers")
for examples of how this should look.
-ed
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 20/20] efx: support pio mapping based on cxl
2024-09-13 17:45 ` Edward Cree
@ 2024-09-16 16:12 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 16:12 UTC (permalink / raw)
To: Edward Cree, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/13/24 18:45, Edward Cree wrote:
> On 07/09/2024 09:18, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> With a device supporting CXL and successfully initialised, use the cxl
>> region to map the memory range and use this mapping for PIO buffers.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> <snip>> diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
>> index cd297e19cddc..c158a1e8d01b 100644
>> --- a/drivers/net/ethernet/sfc/mcdi_pcol.h
>> +++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
>> @@ -16799,6 +16799,9 @@
>> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>>
>> /* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */
>> #define MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160
>> @@ -17303,6 +17306,9 @@
>> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>> /* These bits are reserved for communicating test-specific capabilities to
>> * host-side test software. All production drivers should treat this field as
>> * opaque.
>> @@ -17821,6 +17827,9 @@
>> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>> /* These bits are reserved for communicating test-specific capabilities to
>> * host-side test software. All production drivers should treat this field as
>> * opaque.
>> @@ -18374,6 +18383,9 @@
>> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>> /* These bits are reserved for communicating test-specific capabilities to
>> * host-side test software. All production drivers should treat this field as
>> * opaque.
Hi Ed,
It is really kind of you to comment here, specifically this kind of comment.
> Please, do not make targeted edits to mcdi_pcol.h. Our standard
> process for this file is to regenerate the whole thing from
> smartnic_registry whenever we want to pick up new additions;
> this helps us have certainty and traceability that the driver
> and firmware-side definitions of the MCDI protocol are consistent
> and limit the risk of copy/paste errors etc. introducing
> differences that could cause backwards-compatibility headaches
> later. Ideally the MCDI update should also be a separate commit.
Thank you for telling me ... I already knew it.
FWIW, I have followed the process. I guess you did not see it in our
internal driver repo, but it is in the firmware repo which is the one
that matters here.
You know there are some mcdi_pcol.h definitions not upstreamed, and
therefore it is not possible to just submit the new generated file
upstream but only those bits needed in a patch. I wonder how copy &
paste can be avoided here. Is there a tool we have unbeknownst to me?
BTW, it is almost an standard process those daily team meetings you have
been missing "lately" ... we could have avoided this public exchange ...
> See previous commits to this file, such as
> 1f17708b47a9 ("sfc: update MCDI protocol headers")
> for examples of how this should look.
>
> -ed
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 20/20] efx: support pio mapping based on cxl
2024-09-07 8:18 ` [PATCH v3 20/20] efx: support pio mapping based on cxl alejandro.lucero-palau
2024-09-13 17:45 ` Edward Cree
@ 2024-09-13 17:52 ` Dave Jiang
2024-09-16 16:23 ` Alejandro Lucero Palau
2024-09-13 18:10 ` Jonathan Cameron
2 siblings, 1 reply; 88+ messages in thread
From: Dave Jiang @ 2024-09-13 17:52 UTC (permalink / raw)
To: alejandro.lucero-palau, linux-cxl, netdev, dan.j.williams,
martin.habets, edward.cree, davem, kuba, pabeni, edumazet
Cc: Alejandro Lucero
On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> With a device supporting CXL and successfully initialised, use the cxl
> region to map the memory range and use this mapping for PIO buffers.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> ---
> drivers/net/ethernet/sfc/ef10.c | 32 +++++++++++++++++++++------
> drivers/net/ethernet/sfc/efx_cxl.c | 20 ++++++++++++++++-
> drivers/net/ethernet/sfc/mcdi_pcol.h | 12 ++++++++++
> drivers/net/ethernet/sfc/net_driver.h | 2 ++
> drivers/net/ethernet/sfc/nic.h | 2 ++
> 5 files changed, 60 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
> index 7d69302ffa0a..d4e64cd0f7a4 100644
> --- a/drivers/net/ethernet/sfc/ef10.c
> +++ b/drivers/net/ethernet/sfc/ef10.c
> @@ -24,6 +24,7 @@
> #include <linux/wait.h>
> #include <linux/workqueue.h>
> #include <net/udp_tunnel.h>
> +#include "efx_cxl.h"
>
> /* Hardware control for EF10 architecture including 'Huntington'. */
>
> @@ -177,6 +178,12 @@ static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
> efx->num_mac_stats);
> }
>
> + if (outlen < MC_CMD_GET_CAPABILITIES_V7_OUT_LEN)
> + nic_data->datapath_caps3 = 0;
> + else
> + nic_data->datapath_caps3 = MCDI_DWORD(outbuf,
> + GET_CAPABILITIES_V7_OUT_FLAGS3);
> +
> return 0;
> }
>
> @@ -949,7 +956,7 @@ static void efx_ef10_remove(struct efx_nic *efx)
>
> efx_mcdi_rx_free_indir_table(efx);
>
> - if (nic_data->wc_membase)
> + if (nic_data->wc_membase && !efx->efx_cxl_pio_in_use)
> iounmap(nic_data->wc_membase);
>
> rc = efx_mcdi_free_vis(efx);
> @@ -1263,8 +1270,19 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
> iounmap(efx->membase);
> efx->membase = membase;
>
> - /* Set up the WC mapping if needed */
> - if (wc_mem_map_size) {
> + if (!wc_mem_map_size)
> + return 0;
> +
> + /* Using PIO through CXL mapping? */
> + if ((nic_data->datapath_caps3 &
> + (1 << MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN)) &&
Maybe a FIELD_GET() call would make this cleaner
DJ
> + efx->efx_cxl_pio_initialised) {
> + nic_data->pio_write_base = efx->cxl->ctpio_cxl +
> + (pio_write_vi_base * efx->vi_stride +
> + ER_DZ_TX_PIOBUF - uc_mem_map_size);
> + efx->efx_cxl_pio_in_use = true;
> + } else {
> + /* Using legacy PIO BAR mapping */
> nic_data->wc_membase = ioremap_wc(efx->membase_phys +
> uc_mem_map_size,
> wc_mem_map_size);
> @@ -1279,12 +1297,12 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
> nic_data->wc_membase +
> (pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF -
> uc_mem_map_size);
> -
> - rc = efx_ef10_link_piobufs(efx);
> - if (rc)
> - efx_ef10_free_piobufs(efx);
> }
>
> + rc = efx_ef10_link_piobufs(efx);
> + if (rc)
> + efx_ef10_free_piobufs(efx);
> +
> netif_dbg(efx, probe, efx->net_dev,
> "memory BAR at %pa (virtual %p+%x UC, %p+%x WC)\n",
> &efx->membase_phys, efx->membase, uc_mem_map_size,
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index dd2dbfb8ba15..ef57f833b8a7 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -21,9 +21,9 @@
> int efx_cxl_init(struct efx_nic *efx)
> {
> struct pci_dev *pci_dev = efx->pci_dev;
> + resource_size_t start, end, max = 0;
> struct efx_cxl *cxl;
> struct resource res;
> - resource_size_t max;
> u16 dvsec;
> int rc;
>
> @@ -132,10 +132,27 @@ int efx_cxl_init(struct efx_nic *efx)
> goto err_region;
> }
>
> + rc = cxl_get_region_params(cxl->efx_region, &start, &end);
> + if (rc) {
> + pci_err(pci_dev, "CXL getting regions params failed");
> + goto err_map;
> + }
> +
> + cxl->ctpio_cxl = ioremap(start, end - start);
> + if (!cxl->ctpio_cxl) {
> + pci_err(pci_dev, "CXL ioremap region failed");
> + rc = -EIO;
> + goto err_map;
> + }
> +
> + efx->efx_cxl_pio_initialised = true;
> +
> cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
>
> return 0;
>
> +err_map:
> + cxl_region_detach(cxl->cxled);
> err_region:
> cxl_dpa_free(efx->cxl->cxled);
> err_release:
> @@ -151,6 +168,7 @@ int efx_cxl_init(struct efx_nic *efx)
> void efx_cxl_exit(struct efx_nic *efx)
> {
> if (efx->cxl) {
> + iounmap(efx->cxl->ctpio_cxl);
> cxl_region_detach(efx->cxl->cxled);
> cxl_dpa_free(efx->cxl->cxled);
> cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
> diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
> index cd297e19cddc..c158a1e8d01b 100644
> --- a/drivers/net/ethernet/sfc/mcdi_pcol.h
> +++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
> @@ -16799,6 +16799,9 @@
> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>
> /* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */
> #define MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160
> @@ -17303,6 +17306,9 @@
> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_WIDTH 1
> /* These bits are reserved for communicating test-specific capabilities to
> * host-side test software. All production drivers should treat this field as
> * opaque.
> @@ -17821,6 +17827,9 @@
> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_WIDTH 1
> /* These bits are reserved for communicating test-specific capabilities to
> * host-side test software. All production drivers should treat this field as
> * opaque.
> @@ -18374,6 +18383,9 @@
> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_OFST 148
> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_LBN 17
> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_WIDTH 1
> /* These bits are reserved for communicating test-specific capabilities to
> * host-side test software. All production drivers should treat this field as
> * opaque.
> diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
> index 77261de65e63..893e7841ffb4 100644
> --- a/drivers/net/ethernet/sfc/net_driver.h
> +++ b/drivers/net/ethernet/sfc/net_driver.h
> @@ -967,6 +967,7 @@ struct efx_cxl;
> * @dl_port: devlink port associated with the PF
> * @cxl: details of related cxl objects
> * @efx_cxl_pio_initialised: clx initialization outcome.
> + * @efx_cxl_pio_in_use: PIO using CXL mapping
> * @mem_bar: The BAR that is mapped into membase.
> * @reg_base: Offset from the start of the bar to the function control window.
> * @monitor_work: Hardware monitor workitem
> @@ -1154,6 +1155,7 @@ struct efx_nic {
> struct devlink_port *dl_port;
> struct efx_cxl *cxl;
> bool efx_cxl_pio_initialised;
> + bool efx_cxl_pio_in_use;
> unsigned int mem_bar;
> u32 reg_base;
>
> diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
> index 1db64fc6e909..b7148810acdb 100644
> --- a/drivers/net/ethernet/sfc/nic.h
> +++ b/drivers/net/ethernet/sfc/nic.h
> @@ -151,6 +151,7 @@ enum {
> * @datapath_caps: Capabilities of datapath firmware (FLAGS1 field of
> * %MC_CMD_GET_CAPABILITIES response)
> * @datapath_caps2: Further Capabilities of datapath firmware (FLAGS2 field of
> + * @datapath_caps3: Further Capabilities of datapath firmware (FLAGS3 field of
> * %MC_CMD_GET_CAPABILITIES response)
> * @rx_dpcpu_fw_id: Firmware ID of the RxDPCPU
> * @tx_dpcpu_fw_id: Firmware ID of the TxDPCPU
> @@ -186,6 +187,7 @@ struct efx_ef10_nic_data {
> bool must_check_datapath_caps;
> u32 datapath_caps;
> u32 datapath_caps2;
> + u32 datapath_caps3;
> unsigned int rx_dpcpu_fw_id;
> unsigned int tx_dpcpu_fw_id;
> bool must_probe_vswitching;
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 20/20] efx: support pio mapping based on cxl
2024-09-13 17:52 ` Dave Jiang
@ 2024-09-16 16:23 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 16:23 UTC (permalink / raw)
To: Dave Jiang, alejandro.lucero-palau, linux-cxl, netdev,
dan.j.williams, martin.habets, edward.cree, davem, kuba, pabeni,
edumazet
On 9/13/24 18:52, Dave Jiang wrote:
>
> On 9/7/24 1:18 AM, alejandro.lucero-palau@amd.com wrote:
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> With a device supporting CXL and successfully initialised, use the cxl
>> region to map the memory range and use this mapping for PIO buffers.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
>> ---
>> drivers/net/ethernet/sfc/ef10.c | 32 +++++++++++++++++++++------
>> drivers/net/ethernet/sfc/efx_cxl.c | 20 ++++++++++++++++-
>> drivers/net/ethernet/sfc/mcdi_pcol.h | 12 ++++++++++
>> drivers/net/ethernet/sfc/net_driver.h | 2 ++
>> drivers/net/ethernet/sfc/nic.h | 2 ++
>> 5 files changed, 60 insertions(+), 8 deletions(-)
>>
>> diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
>> index 7d69302ffa0a..d4e64cd0f7a4 100644
>> --- a/drivers/net/ethernet/sfc/ef10.c
>> +++ b/drivers/net/ethernet/sfc/ef10.c
>> @@ -24,6 +24,7 @@
>> #include <linux/wait.h>
>> #include <linux/workqueue.h>
>> #include <net/udp_tunnel.h>
>> +#include "efx_cxl.h"
>>
>> /* Hardware control for EF10 architecture including 'Huntington'. */
>>
>> @@ -177,6 +178,12 @@ static int efx_ef10_init_datapath_caps(struct efx_nic *efx)
>> efx->num_mac_stats);
>> }
>>
>> + if (outlen < MC_CMD_GET_CAPABILITIES_V7_OUT_LEN)
>> + nic_data->datapath_caps3 = 0;
>> + else
>> + nic_data->datapath_caps3 = MCDI_DWORD(outbuf,
>> + GET_CAPABILITIES_V7_OUT_FLAGS3);
>> +
>> return 0;
>> }
>>
>> @@ -949,7 +956,7 @@ static void efx_ef10_remove(struct efx_nic *efx)
>>
>> efx_mcdi_rx_free_indir_table(efx);
>>
>> - if (nic_data->wc_membase)
>> + if (nic_data->wc_membase && !efx->efx_cxl_pio_in_use)
>> iounmap(nic_data->wc_membase);
>>
>> rc = efx_mcdi_free_vis(efx);
>> @@ -1263,8 +1270,19 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
>> iounmap(efx->membase);
>> efx->membase = membase;
>>
>> - /* Set up the WC mapping if needed */
>> - if (wc_mem_map_size) {
>> + if (!wc_mem_map_size)
>> + return 0;
>> +
>> + /* Using PIO through CXL mapping? */
>> + if ((nic_data->datapath_caps3 &
>> + (1 << MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN)) &&
> Maybe a FIELD_GET() call would make this cleaner
>
> DJ
I agree.
Thanks
>
>> + efx->efx_cxl_pio_initialised) {
>> + nic_data->pio_write_base = efx->cxl->ctpio_cxl +
>> + (pio_write_vi_base * efx->vi_stride +
>> + ER_DZ_TX_PIOBUF - uc_mem_map_size);
>> + efx->efx_cxl_pio_in_use = true;
>> + } else {
>> + /* Using legacy PIO BAR mapping */
>> nic_data->wc_membase = ioremap_wc(efx->membase_phys +
>> uc_mem_map_size,
>> wc_mem_map_size);
>> @@ -1279,12 +1297,12 @@ static int efx_ef10_dimension_resources(struct efx_nic *efx)
>> nic_data->wc_membase +
>> (pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF -
>> uc_mem_map_size);
>> -
>> - rc = efx_ef10_link_piobufs(efx);
>> - if (rc)
>> - efx_ef10_free_piobufs(efx);
>> }
>>
>> + rc = efx_ef10_link_piobufs(efx);
>> + if (rc)
>> + efx_ef10_free_piobufs(efx);
>> +
>> netif_dbg(efx, probe, efx->net_dev,
>> "memory BAR at %pa (virtual %p+%x UC, %p+%x WC)\n",
>> &efx->membase_phys, efx->membase, uc_mem_map_size,
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
>> index dd2dbfb8ba15..ef57f833b8a7 100644
>> --- a/drivers/net/ethernet/sfc/efx_cxl.c
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -21,9 +21,9 @@
>> int efx_cxl_init(struct efx_nic *efx)
>> {
>> struct pci_dev *pci_dev = efx->pci_dev;
>> + resource_size_t start, end, max = 0;
>> struct efx_cxl *cxl;
>> struct resource res;
>> - resource_size_t max;
>> u16 dvsec;
>> int rc;
>>
>> @@ -132,10 +132,27 @@ int efx_cxl_init(struct efx_nic *efx)
>> goto err_region;
>> }
>>
>> + rc = cxl_get_region_params(cxl->efx_region, &start, &end);
>> + if (rc) {
>> + pci_err(pci_dev, "CXL getting regions params failed");
>> + goto err_map;
>> + }
>> +
>> + cxl->ctpio_cxl = ioremap(start, end - start);
>> + if (!cxl->ctpio_cxl) {
>> + pci_err(pci_dev, "CXL ioremap region failed");
>> + rc = -EIO;
>> + goto err_map;
>> + }
>> +
>> + efx->efx_cxl_pio_initialised = true;
>> +
>> cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
>>
>> return 0;
>>
>> +err_map:
>> + cxl_region_detach(cxl->cxled);
>> err_region:
>> cxl_dpa_free(efx->cxl->cxled);
>> err_release:
>> @@ -151,6 +168,7 @@ int efx_cxl_init(struct efx_nic *efx)
>> void efx_cxl_exit(struct efx_nic *efx)
>> {
>> if (efx->cxl) {
>> + iounmap(efx->cxl->ctpio_cxl);
>> cxl_region_detach(efx->cxl->cxled);
>> cxl_dpa_free(efx->cxl->cxled);
>> cxl_release_resource(efx->cxl->cxlds, CXL_ACCEL_RES_RAM);
>> diff --git a/drivers/net/ethernet/sfc/mcdi_pcol.h b/drivers/net/ethernet/sfc/mcdi_pcol.h
>> index cd297e19cddc..c158a1e8d01b 100644
>> --- a/drivers/net/ethernet/sfc/mcdi_pcol.h
>> +++ b/drivers/net/ethernet/sfc/mcdi_pcol.h
>> @@ -16799,6 +16799,9 @@
>> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>> #define MC_CMD_GET_CAPABILITIES_V7_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>>
>> /* MC_CMD_GET_CAPABILITIES_V8_OUT msgresponse */
>> #define MC_CMD_GET_CAPABILITIES_V8_OUT_LEN 160
>> @@ -17303,6 +17306,9 @@
>> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>> #define MC_CMD_GET_CAPABILITIES_V8_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define MC_CMD_GET_CAPABILITIES_V8_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>> /* These bits are reserved for communicating test-specific capabilities to
>> * host-side test software. All production drivers should treat this field as
>> * opaque.
>> @@ -17821,6 +17827,9 @@
>> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>> #define MC_CMD_GET_CAPABILITIES_V9_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define MC_CMD_GET_CAPABILITIES_V9_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>> /* These bits are reserved for communicating test-specific capabilities to
>> * host-side test software. All production drivers should treat this field as
>> * opaque.
>> @@ -18374,6 +18383,9 @@
>> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_OFST 148
>> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_LBN 14
>> #define MC_CMD_GET_CAPABILITIES_V10_OUT_DYNAMIC_MPORT_JOURNAL_WIDTH 1
>> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_OFST 148
>> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_LBN 17
>> +#define MC_CMD_GET_CAPABILITIES_V10_OUT_CXL_CONFIG_ENABLE_WIDTH 1
>> /* These bits are reserved for communicating test-specific capabilities to
>> * host-side test software. All production drivers should treat this field as
>> * opaque.
>> diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
>> index 77261de65e63..893e7841ffb4 100644
>> --- a/drivers/net/ethernet/sfc/net_driver.h
>> +++ b/drivers/net/ethernet/sfc/net_driver.h
>> @@ -967,6 +967,7 @@ struct efx_cxl;
>> * @dl_port: devlink port associated with the PF
>> * @cxl: details of related cxl objects
>> * @efx_cxl_pio_initialised: clx initialization outcome.
>> + * @efx_cxl_pio_in_use: PIO using CXL mapping
>> * @mem_bar: The BAR that is mapped into membase.
>> * @reg_base: Offset from the start of the bar to the function control window.
>> * @monitor_work: Hardware monitor workitem
>> @@ -1154,6 +1155,7 @@ struct efx_nic {
>> struct devlink_port *dl_port;
>> struct efx_cxl *cxl;
>> bool efx_cxl_pio_initialised;
>> + bool efx_cxl_pio_in_use;
>> unsigned int mem_bar;
>> u32 reg_base;
>>
>> diff --git a/drivers/net/ethernet/sfc/nic.h b/drivers/net/ethernet/sfc/nic.h
>> index 1db64fc6e909..b7148810acdb 100644
>> --- a/drivers/net/ethernet/sfc/nic.h
>> +++ b/drivers/net/ethernet/sfc/nic.h
>> @@ -151,6 +151,7 @@ enum {
>> * @datapath_caps: Capabilities of datapath firmware (FLAGS1 field of
>> * %MC_CMD_GET_CAPABILITIES response)
>> * @datapath_caps2: Further Capabilities of datapath firmware (FLAGS2 field of
>> + * @datapath_caps3: Further Capabilities of datapath firmware (FLAGS3 field of
>> * %MC_CMD_GET_CAPABILITIES response)
>> * @rx_dpcpu_fw_id: Firmware ID of the RxDPCPU
>> * @tx_dpcpu_fw_id: Firmware ID of the TxDPCPU
>> @@ -186,6 +187,7 @@ struct efx_ef10_nic_data {
>> bool must_check_datapath_caps;
>> u32 datapath_caps;
>> u32 datapath_caps2;
>> + u32 datapath_caps3;
>> unsigned int rx_dpcpu_fw_id;
>> unsigned int tx_dpcpu_fw_id;
>> bool must_probe_vswitching;
^ permalink raw reply [flat|nested] 88+ messages in thread
* Re: [PATCH v3 20/20] efx: support pio mapping based on cxl
2024-09-07 8:18 ` [PATCH v3 20/20] efx: support pio mapping based on cxl alejandro.lucero-palau
2024-09-13 17:45 ` Edward Cree
2024-09-13 17:52 ` Dave Jiang
@ 2024-09-13 18:10 ` Jonathan Cameron
2024-09-16 16:23 ` Alejandro Lucero Palau
2 siblings, 1 reply; 88+ messages in thread
From: Jonathan Cameron @ 2024-09-13 18:10 UTC (permalink / raw)
To: alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet, Alejandro Lucero
On Sat, 7 Sep 2024 09:18:36 +0100
alejandro.lucero-palau@amd.com wrote:
> From: Alejandro Lucero <alucerop@amd.com>
>
> With a device supporting CXL and successfully initialised, use the cxl
> region to map the memory range and use this mapping for PIO buffers.
>
> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
One trivial thing.
> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
> index dd2dbfb8ba15..ef57f833b8a7 100644
> --- a/drivers/net/ethernet/sfc/efx_cxl.c
> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
> @@ -21,9 +21,9 @@
> int efx_cxl_init(struct efx_nic *efx)
> {
> struct pci_dev *pci_dev = efx->pci_dev;
> + resource_size_t start, end, max = 0;
> struct efx_cxl *cxl;
> struct resource res;
> - resource_size_t max;
> u16 dvsec;
> int rc;
>
> @@ -132,10 +132,27 @@ int efx_cxl_init(struct efx_nic *efx)
> goto err_region;
> }
>
> + rc = cxl_get_region_params(cxl->efx_region, &start, &end);
> + if (rc) {
> + pci_err(pci_dev, "CXL getting regions params failed");
> + goto err_map;
> + }
> +
> + cxl->ctpio_cxl = ioremap(start, end - start);
> + if (!cxl->ctpio_cxl) {
> + pci_err(pci_dev, "CXL ioremap region failed");
> + rc = -EIO;
> + goto err_map;
> + }
> +
> + efx->efx_cxl_pio_initialised = true;
> +
> cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
>
> return 0;
>
> +err_map:
> + cxl_region_detach(cxl->cxled);
Odd looking indent.
> err_region:
> cxl_dpa_free(efx->cxl->cxled);
> err_release:
^ permalink raw reply [flat|nested] 88+ messages in thread* Re: [PATCH v3 20/20] efx: support pio mapping based on cxl
2024-09-13 18:10 ` Jonathan Cameron
@ 2024-09-16 16:23 ` Alejandro Lucero Palau
0 siblings, 0 replies; 88+ messages in thread
From: Alejandro Lucero Palau @ 2024-09-16 16:23 UTC (permalink / raw)
To: Jonathan Cameron, alejandro.lucero-palau
Cc: linux-cxl, netdev, dan.j.williams, martin.habets, edward.cree,
davem, kuba, pabeni, edumazet
On 9/13/24 19:10, Jonathan Cameron wrote:
> On Sat, 7 Sep 2024 09:18:36 +0100
> alejandro.lucero-palau@amd.com wrote:
>
>> From: Alejandro Lucero <alucerop@amd.com>
>>
>> With a device supporting CXL and successfully initialised, use the cxl
>> region to map the memory range and use this mapping for PIO buffers.
>>
>> Signed-off-by: Alejandro Lucero <alucerop@amd.com>
> One trivial thing.
>
>> diff --git a/drivers/net/ethernet/sfc/efx_cxl.c b/drivers/net/ethernet/sfc/efx_cxl.c
>> index dd2dbfb8ba15..ef57f833b8a7 100644
>> --- a/drivers/net/ethernet/sfc/efx_cxl.c
>> +++ b/drivers/net/ethernet/sfc/efx_cxl.c
>> @@ -21,9 +21,9 @@
>> int efx_cxl_init(struct efx_nic *efx)
>> {
>> struct pci_dev *pci_dev = efx->pci_dev;
>> + resource_size_t start, end, max = 0;
>> struct efx_cxl *cxl;
>> struct resource res;
>> - resource_size_t max;
>> u16 dvsec;
>> int rc;
>>
>> @@ -132,10 +132,27 @@ int efx_cxl_init(struct efx_nic *efx)
>> goto err_region;
>> }
>>
>> + rc = cxl_get_region_params(cxl->efx_region, &start, &end);
>> + if (rc) {
>> + pci_err(pci_dev, "CXL getting regions params failed");
>> + goto err_map;
>> + }
>> +
>> + cxl->ctpio_cxl = ioremap(start, end - start);
>> + if (!cxl->ctpio_cxl) {
>> + pci_err(pci_dev, "CXL ioremap region failed");
>> + rc = -EIO;
>> + goto err_map;
>> + }
>> +
>> + efx->efx_cxl_pio_initialised = true;
>> +
>> cxl_release_endpoint(cxl->cxlmd, cxl->endpoint);
>>
>> return 0;
>>
>> +err_map:
>> + cxl_region_detach(cxl->cxled);
> Odd looking indent.
I'll fix it.
Thanks!
>> err_region:
>> cxl_dpa_free(efx->cxl->cxled);
>> err_release:
^ permalink raw reply [flat|nested] 88+ messages in thread