[PATCH v2 0/3] cxl: Fix nvdimm_bus race for nvdimm devices

[PATCH v2 0/3] cxl: Fix nvdimm_bus race for nvdimm devices

[Dave Jiang]

v2:
- Try a different approach

A small series to address a race for nvdimm_bus object when using the
nvdimm_bridge object.

Dave Jiang (3):
  cxl: Move devm_cxl_add_nvdimm_bridge() to cxl_pmem.ko
  cxl: Fix race of nvdimm_bus object when creating nvdimm objects
  cxl: Clean up cxl_nvdimm devices when removing nvdimm_bus

 drivers/cxl/core/core.h |   1 -
 drivers/cxl/core/pmem.c | 105 ++++++++--------------------------------
 drivers/cxl/cxl.h       |   2 +
 drivers/cxl/pmem.c      | 102 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 125 insertions(+), 85 deletions(-)


base-commit: 63fbf275fa9f18f7020fb8acf54fa107e51d0f23
-- 
2.52.0

[PATCH v2 1/3] cxl: Move devm_cxl_add_nvdimm_bridge() to cxl_pmem.ko

[Dave Jiang]

Moving devm_cxl_add_nvdimm_bridge() and related functions to
drivers/cxl/cxl_pmem.c, so that cxl_pmem can export a symbol that gives
cxl_acpi a depedency on cxl_pmem kernel module. This is a prepatory patch
to resolve the issue of a race for nvdimm_bus object that is created
during cxl_acpi_probe().

No functional changes besides moving code.

Suggested-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 drivers/cxl/core/core.h |  1 -
 drivers/cxl/core/pmem.c | 86 ++---------------------------------------
 drivers/cxl/cxl.h       |  1 +
 drivers/cxl/pmem.c      | 84 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 88 insertions(+), 84 deletions(-)

diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
index 007b8aff0238..8ee0c3bb2065 100644
--- a/drivers/cxl/core/core.h
+++ b/drivers/cxl/core/core.h
@@ -7,7 +7,6 @@
 #include <cxl/mailbox.h>
 #include <linux/rwsem.h>
 
-extern const struct device_type cxl_nvdimm_bridge_type;
 extern const struct device_type cxl_nvdimm_type;
 extern const struct device_type cxl_pmu_type;
 
diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c
index e7b1e6fa0ea0..50557d71a45f 100644
--- a/drivers/cxl/core/pmem.c
+++ b/drivers/cxl/core/pmem.c
@@ -21,7 +21,8 @@
  * operations, for example, namespace label access commands.
  */
 
-static DEFINE_IDA(cxl_nvdimm_bridge_ida);
+DEFINE_IDA(cxl_nvdimm_bridge_ida);
+EXPORT_SYMBOL_NS_GPL(cxl_nvdimm_bridge_ida, "CXL");
 
 static void cxl_nvdimm_bridge_release(struct device *dev)
 {
@@ -41,6 +42,7 @@ const struct device_type cxl_nvdimm_bridge_type = {
 	.release = cxl_nvdimm_bridge_release,
 	.groups = cxl_nvdimm_bridge_attribute_groups,
 };
+EXPORT_SYMBOL_NS_GPL(cxl_nvdimm_bridge_type, "CXL");
 
 struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev)
 {
@@ -75,88 +77,6 @@ struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_port *port)
 }
 EXPORT_SYMBOL_NS_GPL(cxl_find_nvdimm_bridge, "CXL");
 
-static struct lock_class_key cxl_nvdimm_bridge_key;
-
-static struct cxl_nvdimm_bridge *cxl_nvdimm_bridge_alloc(struct cxl_port *port)
-{
-	struct cxl_nvdimm_bridge *cxl_nvb;
-	struct device *dev;
-	int rc;
-
-	cxl_nvb = kzalloc(sizeof(*cxl_nvb), GFP_KERNEL);
-	if (!cxl_nvb)
-		return ERR_PTR(-ENOMEM);
-
-	rc = ida_alloc(&cxl_nvdimm_bridge_ida, GFP_KERNEL);
-	if (rc < 0)
-		goto err;
-	cxl_nvb->id = rc;
-
-	dev = &cxl_nvb->dev;
-	cxl_nvb->port = port;
-	device_initialize(dev);
-	lockdep_set_class(&dev->mutex, &cxl_nvdimm_bridge_key);
-	device_set_pm_not_required(dev);
-	dev->parent = &port->dev;
-	dev->bus = &cxl_bus_type;
-	dev->type = &cxl_nvdimm_bridge_type;
-
-	return cxl_nvb;
-
-err:
-	kfree(cxl_nvb);
-	return ERR_PTR(rc);
-}
-
-static void unregister_nvb(void *_cxl_nvb)
-{
-	struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
-
-	device_unregister(&cxl_nvb->dev);
-}
-
-/**
- * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology
- * @host: platform firmware root device
- * @port: CXL port at the root of a CXL topology
- *
- * Return: bridge device that can host cxl_nvdimm objects
- */
-struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
-						     struct cxl_port *port)
-{
-	struct cxl_nvdimm_bridge *cxl_nvb;
-	struct device *dev;
-	int rc;
-
-	if (!IS_ENABLED(CONFIG_CXL_PMEM))
-		return ERR_PTR(-ENXIO);
-
-	cxl_nvb = cxl_nvdimm_bridge_alloc(port);
-	if (IS_ERR(cxl_nvb))
-		return cxl_nvb;
-
-	dev = &cxl_nvb->dev;
-	rc = dev_set_name(dev, "nvdimm-bridge%d", cxl_nvb->id);
-	if (rc)
-		goto err;
-
-	rc = device_add(dev);
-	if (rc)
-		goto err;
-
-	rc = devm_add_action_or_reset(host, unregister_nvb, cxl_nvb);
-	if (rc)
-		return ERR_PTR(rc);
-
-	return cxl_nvb;
-
-err:
-	put_device(dev);
-	return ERR_PTR(rc);
-}
-EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL");
-
 static void cxl_nvdimm_release(struct device *dev)
 {
 	struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev);
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 04c673e7cdb0..3b79d5c4765a 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -877,6 +877,7 @@ int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds,
 bool is_cxl_region(struct device *dev);
 
 extern const struct bus_type cxl_bus_type;
+extern const struct device_type cxl_nvdimm_bridge_type;
 
 /*
  * Note, add_dport() is expressly for the cxl_port driver. TODO: investigate a
diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
index e197883690ef..6f010b3cce44 100644
--- a/drivers/cxl/pmem.c
+++ b/drivers/cxl/pmem.c
@@ -13,6 +13,90 @@
 
 static __read_mostly DECLARE_BITMAP(exclusive_cmds, CXL_MEM_COMMAND_ID_MAX);
 
+extern struct ida cxl_nvdimm_bridge_ida;
+
+static struct lock_class_key cxl_nvdimm_bridge_key;
+
+static struct cxl_nvdimm_bridge *cxl_nvdimm_bridge_alloc(struct cxl_port *port)
+{
+	struct cxl_nvdimm_bridge *cxl_nvb;
+	struct device *dev;
+	int rc;
+
+	cxl_nvb = kzalloc(sizeof(*cxl_nvb), GFP_KERNEL);
+	if (!cxl_nvb)
+		return ERR_PTR(-ENOMEM);
+
+	rc = ida_alloc(&cxl_nvdimm_bridge_ida, GFP_KERNEL);
+	if (rc < 0)
+		goto err;
+	cxl_nvb->id = rc;
+
+	dev = &cxl_nvb->dev;
+	cxl_nvb->port = port;
+	device_initialize(dev);
+	lockdep_set_class(&dev->mutex, &cxl_nvdimm_bridge_key);
+	device_set_pm_not_required(dev);
+	dev->parent = &port->dev;
+	dev->bus = &cxl_bus_type;
+	dev->type = &cxl_nvdimm_bridge_type;
+
+	return cxl_nvb;
+
+err:
+	kfree(cxl_nvb);
+	return ERR_PTR(rc);
+}
+
+static void unregister_nvb(void *_cxl_nvb)
+{
+	struct cxl_nvdimm_bridge *cxl_nvb = _cxl_nvb;
+
+	device_unregister(&cxl_nvb->dev);
+}
+
+/**
+ * devm_cxl_add_nvdimm_bridge() - add the root of a LIBNVDIMM topology
+ * @host: platform firmware root device
+ * @port: CXL port at the root of a CXL topology
+ *
+ * Return: bridge device that can host cxl_nvdimm objects
+ */
+struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
+						     struct cxl_port *port)
+{
+	struct cxl_nvdimm_bridge *cxl_nvb;
+	struct device *dev;
+	int rc;
+
+	if (!IS_ENABLED(CONFIG_CXL_PMEM))
+		return ERR_PTR(-ENXIO);
+
+	cxl_nvb = cxl_nvdimm_bridge_alloc(port);
+	if (IS_ERR(cxl_nvb))
+		return cxl_nvb;
+
+	dev = &cxl_nvb->dev;
+	rc = dev_set_name(dev, "nvdimm-bridge%d", cxl_nvb->id);
+	if (rc)
+		goto err;
+
+	rc = device_add(dev);
+	if (rc)
+		goto err;
+
+	rc = devm_add_action_or_reset(host, unregister_nvb, cxl_nvb);
+	if (rc)
+		return ERR_PTR(rc);
+
+	return cxl_nvb;
+
+err:
+	put_device(dev);
+	return ERR_PTR(rc);
+}
+EXPORT_SYMBOL_NS_GPL(devm_cxl_add_nvdimm_bridge, "CXL");
+
 static void clear_exclusive(void *mds)
 {
 	clear_exclusive_cxl_commands(mds, exclusive_cmds);
-- 
2.52.0

Re: [PATCH v2 1/3] cxl: Move devm_cxl_add_nvdimm_bridge() to cxl_pmem.ko

[dan.j.williams]

Dave Jiang wrote:
> Moving devm_cxl_add_nvdimm_bridge() and related functions to
> drivers/cxl/cxl_pmem.c, so that cxl_pmem can export a symbol that gives
> cxl_acpi a depedency on cxl_pmem kernel module. This is a prepatory patch
> to resolve the issue of a race for nvdimm_bus object that is created
> during cxl_acpi_probe().
> 
> No functional changes besides moving code.
> 
> Suggested-by: Dan Williams <dan.j.williams@intel.com>
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
> ---
>  drivers/cxl/core/core.h |  1 -
>  drivers/cxl/core/pmem.c | 86 ++---------------------------------------
>  drivers/cxl/cxl.h       |  1 +
>  drivers/cxl/pmem.c      | 84 ++++++++++++++++++++++++++++++++++++++++
>  4 files changed, 88 insertions(+), 84 deletions(-)
> 
> diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h
> index 007b8aff0238..8ee0c3bb2065 100644
> --- a/drivers/cxl/core/core.h
> +++ b/drivers/cxl/core/core.h
> @@ -7,7 +7,6 @@
>  #include <cxl/mailbox.h>
>  #include <linux/rwsem.h>
>  
> -extern const struct device_type cxl_nvdimm_bridge_type;
>  extern const struct device_type cxl_nvdimm_type;
>  extern const struct device_type cxl_pmu_type;
>  
> diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c
> index e7b1e6fa0ea0..50557d71a45f 100644
> --- a/drivers/cxl/core/pmem.c
> +++ b/drivers/cxl/core/pmem.c
> @@ -21,7 +21,8 @@
>   * operations, for example, namespace label access commands.
>   */
>  
> -static DEFINE_IDA(cxl_nvdimm_bridge_ida);
> +DEFINE_IDA(cxl_nvdimm_bridge_ida);
> +EXPORT_SYMBOL_NS_GPL(cxl_nvdimm_bridge_ida, "CXL");
>  
>  static void cxl_nvdimm_bridge_release(struct device *dev)
>  {
> @@ -41,6 +42,7 @@ const struct device_type cxl_nvdimm_bridge_type = {
>  	.release = cxl_nvdimm_bridge_release,
>  	.groups = cxl_nvdimm_bridge_attribute_groups,
>  };
> +EXPORT_SYMBOL_NS_GPL(cxl_nvdimm_bridge_type, "CXL");

-ETOOMANYEXPORTS

What about just copying the approach of:

1f1cb7f0c255 cxl/mem: Arrange for always-synchronous memdev attach

...that did:

struct cxl_memdev *devm_cxl_add_memdev(struct cxl_dev_state *cxlds,
                                       const struct cxl_memdev_attach *attach)
{
        return __devm_cxl_add_memdev(cxlds, attach);
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_add_memdev, "CXL");

In other words keep all the core infra in the core and only have a
private export from the core to the driver module.

[PATCH v2 2/3] cxl: Fix race of nvdimm_bus object when creating nvdimm objects

[Dave Jiang]

Found issue during running of cxl-translate.sh unit test. Adding a 3s
sleep right before the test seems to make the issue reproduce fairly
consistently. The cxl_translate module has dependency on cxl_acpi and
causes orphaned nvdimm objects to reprobe after cxl_acpi is removed.
The nvdimm_bus object is registered by the cxl_nvb object when
cxl_acpi_probe() is called. With the nvdimm_bus object missing,
__nd_device_register() will trigger NULL pointer dereference when
accessing the dev->parent that points to &nvdimm_bus->dev.

[  192.884510] BUG: kernel NULL pointer dereference, address: 000000000000006c
[  192.895383] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS edk2-20250812-19.fc42 08/12/2025
[  192.897721] Workqueue: cxl_port cxl_bus_rescan_queue [cxl_core]
[  192.899459] RIP: 0010:kobject_get+0xc/0x90
[  192.924871] Call Trace:
[  192.925959]  <TASK>
[  192.926976]  ? pm_runtime_init+0xb9/0xe0
[  192.929712]  __nd_device_register.part.0+0x4d/0xc0 [libnvdimm]
[  192.933314]  __nvdimm_create+0x206/0x290 [libnvdimm]
[  192.936662]  cxl_nvdimm_probe+0x119/0x1d0 [cxl_pmem]
[  192.940245]  cxl_bus_probe+0x1a/0x60 [cxl_core]
[  192.943349]  really_probe+0xde/0x380
[  192.945614]  ? _raw_spin_unlock_irq+0x18/0x40
[  192.948402]  ? __pfx___device_attach_driver+0x10/0x10
[  192.951407]  __driver_probe_device+0xc0/0x150
[  192.953997]  driver_probe_device+0x1f/0xa0
[  192.956456]  __device_attach_driver+0x85/0x130
[  192.959231]  ? _raw_spin_unlock+0x12/0x30
[  192.961615]  bus_for_each_drv+0x6c/0xb0
[  192.963935]  __device_attach+0xad/0x1c0
[  192.966213]  ? __pfx_cxl_rescan_attach+0x10/0x10 [cxl_core]
[  192.969350]  cxl_rescan_attach+0xa/0x20 [cxl_core]
[  192.972063]  bus_for_each_dev+0x63/0xa0
[  192.974367]  process_one_work+0x166/0x340
[  192.976758]  worker_thread+0x258/0x3a0
[  192.979011]  ? __pfx_worker_thread+0x10/0x10
[  192.981487]  kthread+0x108/0x220
[  192.983503]  ? __pfx_kthread+0x10/0x10
[  192.985335]  ? __pfx_kthread+0x10/0x10
[  192.986148]  ret_from_fork+0x246/0x280
[  192.987018]  ? __pfx_kthread+0x10/0x10
[  192.987891]  ret_from_fork_asm+0x1a/0x30
[  192.988768]  </TASK>
[  192.989359] Modules linked in: cxl_pmem(O) cxl_acpi(O-) kmem device_dax dax_cxl dax_pmem nd_pmem nd_btt cxl_mock_mem(O) dax_hmem cxl_pci nd_e820 nfit cxl_mem(O) cxl_port(O) cxl_mock(O) libnvdimm cxl_core(O) fwctl [last unloaded: cxl_translate(O)]
[  192.993533] CR2: 000000000000006c
[  192.994417] ---[ end trace 0000000000000000 ]---

1. Set probe_type of cxl_nvb to PROBE_FORCE_SYNCHRONOUS to ensure the
   driver is probed synchronously when add_device() is called.
2. Add a check after add_device() for cxl_nvb to ensure the driver is
   bound, which also means the nvdimm_bus object is registered.
3. Add a check in __nd_device_register() to avoid NULL pointer
   dereference when nvdimm_bus is not registered. Hold the uport_dev
   lock of the cxl rootport to sychronize between cxl_acpi_probe() and
   cxl_devm_cxl_add_nvdimm(). Also hold the cxl_nvb->dev lock to ensure
   a consistent state of if the driver is bound (or not).
4. Add a check for nvdimm_bus in cxl_nvdimm_probe() to avoid accessing
   without nvdimm_bus for unbound nvdimm object reprobes.

This fix also requies the commit that moves devm_cxl_add_nvdimm_bridge()
from core/pmem.c to pmem.c.

Fixes: 21083f51521f ("cxl/pmem: Register 'pmem' / cxl_nvdimm devices")
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 drivers/cxl/core/pmem.c | 13 +++++++++++++
 drivers/cxl/pmem.c      | 13 +++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c
index 50557d71a45f..f7f784047450 100644
--- a/drivers/cxl/core/pmem.c
+++ b/drivers/cxl/core/pmem.c
@@ -175,6 +175,19 @@ int devm_cxl_add_nvdimm(struct device *host, struct cxl_port *port,
 	if (!cxl_nvb)
 		return -ENODEV;
 
+	/*
+	 * Take the uport_dev lock to guard against race of nvdimm_bus object.
+	 * cxl_acpi_probe() registers the nvdimm_bus and is done under the
+	 * root port uport_dev lock.
+	 *
+	 * Take the cxl_nvb device lock to ensure that cxl_nvb driver is in a
+	 * consistent state. And the driver registers nvdimm_bus.
+	 */
+	guard(device)(cxl_nvb->port->uport_dev);
+	guard(device)(&cxl_nvb->dev);
+	if (!cxl_nvb->nvdimm_bus)
+		return -ENODEV;
+
 	cxl_nvd = cxl_nvdimm_alloc(cxl_nvb, cxlmd);
 	if (IS_ERR(cxl_nvd)) {
 		rc = PTR_ERR(cxl_nvd);
diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
index 6f010b3cce44..0b200be1fd36 100644
--- a/drivers/cxl/pmem.c
+++ b/drivers/cxl/pmem.c
@@ -85,6 +85,15 @@ struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
 	if (rc)
 		goto err;
 
+	/* Ensure that cxl_nvb driver has been bound since it is synchronous. */
+	device_lock(dev);
+	if (!dev->driver) {
+		device_unlock(dev);
+		unregister_nvb(cxl_nvb);
+		return ERR_PTR(-ENODEV);
+	}
+	device_unlock(dev);
+
 	rc = devm_add_action_or_reset(host, unregister_nvb, cxl_nvb);
 	if (rc)
 		return ERR_PTR(rc);
@@ -213,6 +222,9 @@ static int cxl_nvdimm_probe(struct device *dev)
 	struct nvdimm *nvdimm;
 	int rc;
 
+	if (!cxl_nvb->nvdimm_bus)
+		return -ENODEV;
+
 	set_exclusive_cxl_commands(mds, exclusive_cmds);
 	rc = devm_add_action_or_reset(dev, clear_exclusive, mds);
 	if (rc)
@@ -437,6 +449,7 @@ static struct cxl_driver cxl_nvdimm_bridge_driver = {
 	.probe = cxl_nvdimm_bridge_probe,
 	.id = CXL_DEVICE_NVDIMM_BRIDGE,
 	.drv = {
+		.probe_type = PROBE_FORCE_SYNCHRONOUS,
 		.suppress_bind_attrs = true,
 	},
 };
-- 
2.52.0

Re: [PATCH v2 2/3] cxl: Fix race of nvdimm_bus object when creating nvdimm objects

[dan.j.williams]

Dave Jiang wrote:
> Found issue during running of cxl-translate.sh unit test. Adding a 3s
> sleep right before the test seems to make the issue reproduce fairly
> consistently. The cxl_translate module has dependency on cxl_acpi and
> causes orphaned nvdimm objects to reprobe after cxl_acpi is removed.
> The nvdimm_bus object is registered by the cxl_nvb object when
> cxl_acpi_probe() is called. With the nvdimm_bus object missing,
> __nd_device_register() will trigger NULL pointer dereference when
> accessing the dev->parent that points to &nvdimm_bus->dev.
> 

Please trim backtraces of useless info, something like:

 BUG: kernel NULL pointer dereference, address: 000000000000006c
 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS edk2-20250812-19.fc42 08/12/2025
 Workqueue: cxl_port cxl_bus_rescan_queue [cxl_core]
 RIP: 0010:kobject_get+0xc/0x90
 Call Trace:
  <TASK>
  ? pm_runtime_init+0xb9/0xe0
  __nd_device_register.part.0+0x4d/0xc0 [libnvdimm]
  __nvdimm_create+0x206/0x290 [libnvdimm]
  cxl_nvdimm_probe+0x119/0x1d0 [cxl_pmem]
  cxl_bus_probe+0x1a/0x60 [cxl_core]
  really_probe+0xde/0x380
> 
> 1. Set probe_type of cxl_nvb to PROBE_FORCE_SYNCHRONOUS to ensure the
>    driver is probed synchronously when add_device() is called.
> 2. Add a check after add_device() for cxl_nvb to ensure the driver is
>    bound, which also means the nvdimm_bus object is registered.
> 3. Add a check in __nd_device_register() to avoid NULL pointer
>    dereference when nvdimm_bus is not registered.

This is not done in this patch and not needed.

>    Hold the uport_dev
>    lock of the cxl rootport to sychronize between cxl_acpi_probe() and
>    cxl_devm_cxl_add_nvdimm(). Also hold the cxl_nvb->dev lock to ensure
>    a consistent state of if the driver is bound (or not).
> 4. Add a check for nvdimm_bus in cxl_nvdimm_probe() to avoid accessing
>    without nvdimm_bus for unbound nvdimm object reprobes.

See below, I would have expected that deleting stale nvdimm devices
would also be part of this patch.

> 
> This fix also requies the commit that moves devm_cxl_add_nvdimm_bridge()
> from core/pmem.c to pmem.c.
> 
> Fixes: 21083f51521f ("cxl/pmem: Register 'pmem' / cxl_nvdimm devices")
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
> ---
>  drivers/cxl/core/pmem.c | 13 +++++++++++++
>  drivers/cxl/pmem.c      | 13 +++++++++++++
>  2 files changed, 26 insertions(+)
> 
> diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c
> index 50557d71a45f..f7f784047450 100644
> --- a/drivers/cxl/core/pmem.c
> +++ b/drivers/cxl/core/pmem.c
> @@ -175,6 +175,19 @@ int devm_cxl_add_nvdimm(struct device *host, struct cxl_port *port,
>  	if (!cxl_nvb)
>  		return -ENODEV;
>  
> +	/*
> +	 * Take the uport_dev lock to guard against race of nvdimm_bus object.
> +	 * cxl_acpi_probe() registers the nvdimm_bus and is done under the
> +	 * root port uport_dev lock.
> +	 *
> +	 * Take the cxl_nvb device lock to ensure that cxl_nvb driver is in a
> +	 * consistent state. And the driver registers nvdimm_bus.
> +	 */
> +	guard(device)(cxl_nvb->port->uport_dev);
> +	guard(device)(&cxl_nvb->dev);
> +	if (!cxl_nvb->nvdimm_bus)
> +		return -ENODEV;
> +
>  	cxl_nvd = cxl_nvdimm_alloc(cxl_nvb, cxlmd);
>  	if (IS_ERR(cxl_nvd)) {
>  		rc = PTR_ERR(cxl_nvd);
> diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
> index 6f010b3cce44..0b200be1fd36 100644
> --- a/drivers/cxl/pmem.c
> +++ b/drivers/cxl/pmem.c
> @@ -85,6 +85,15 @@ struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
>  	if (rc)
>  		goto err;
>  
> +	/* Ensure that cxl_nvb driver has been bound since it is synchronous. */
> +	device_lock(dev);
> +	if (!dev->driver) {
> +		device_unlock(dev);
> +		unregister_nvb(cxl_nvb);
> +		return ERR_PTR(-ENODEV);
> +	}
> +	device_unlock(dev);

This can be a cxl_nvdimm_bridge_failed_attach() helper what uses
guard().

> +
>  	rc = devm_add_action_or_reset(host, unregister_nvb, cxl_nvb);
>  	if (rc)
>  		return ERR_PTR(rc);
> @@ -213,6 +222,9 @@ static int cxl_nvdimm_probe(struct device *dev)
>  	struct nvdimm *nvdimm;
>  	int rc;
>  
> +	if (!cxl_nvb->nvdimm_bus)
> +		return -ENODEV;

If I understand the failure scenario correctly, this is racy. What keeps
nvdimm_bus valid for the duration of cxl_nvdimm probe?

This is why I think the cxl_nvdimm_bridge disable path needs to evacuate
all prior cxl_nvdimm objects that saw nvdimm_bus live at their
registration time.

Re: [PATCH v2 2/3] cxl: Fix race of nvdimm_bus object when creating nvdimm objects

[Dave Jiang]

On 2/11/26 4:55 PM, dan.j.williams@intel.com wrote:
> Dave Jiang wrote:
>> Found issue during running of cxl-translate.sh unit test. Adding a 3s
>> sleep right before the test seems to make the issue reproduce fairly
>> consistently. The cxl_translate module has dependency on cxl_acpi and
>> causes orphaned nvdimm objects to reprobe after cxl_acpi is removed.
>> The nvdimm_bus object is registered by the cxl_nvb object when
>> cxl_acpi_probe() is called. With the nvdimm_bus object missing,
>> __nd_device_register() will trigger NULL pointer dereference when
>> accessing the dev->parent that points to &nvdimm_bus->dev.
>>
> 
> Please trim backtraces of useless info, something like:
> 
>  BUG: kernel NULL pointer dereference, address: 000000000000006c
>  Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS edk2-20250812-19.fc42 08/12/2025
>  Workqueue: cxl_port cxl_bus_rescan_queue [cxl_core]
>  RIP: 0010:kobject_get+0xc/0x90
>  Call Trace:
>   <TASK>
>   ? pm_runtime_init+0xb9/0xe0
>   __nd_device_register.part.0+0x4d/0xc0 [libnvdimm]
>   __nvdimm_create+0x206/0x290 [libnvdimm]
>   cxl_nvdimm_probe+0x119/0x1d0 [cxl_pmem]
>   cxl_bus_probe+0x1a/0x60 [cxl_core]
>   really_probe+0xde/0x380
>>
>> 1. Set probe_type of cxl_nvb to PROBE_FORCE_SYNCHRONOUS to ensure the
>>    driver is probed synchronously when add_device() is called.
>> 2. Add a check after add_device() for cxl_nvb to ensure the driver is
>>    bound, which also means the nvdimm_bus object is registered.
>> 3. Add a check in __nd_device_register() to avoid NULL pointer
>>    dereference when nvdimm_bus is not registered.
> 
> This is not done in this patch and not needed.
> 
>>    Hold the uport_dev
>>    lock of the cxl rootport to sychronize between cxl_acpi_probe() and
>>    cxl_devm_cxl_add_nvdimm(). Also hold the cxl_nvb->dev lock to ensure
>>    a consistent state of if the driver is bound (or not).
>> 4. Add a check for nvdimm_bus in cxl_nvdimm_probe() to avoid accessing
>>    without nvdimm_bus for unbound nvdimm object reprobes.
> 
> See below, I would have expected that deleting stale nvdimm devices
> would also be part of this patch.
> 
>>
>> This fix also requies the commit that moves devm_cxl_add_nvdimm_bridge()
>> from core/pmem.c to pmem.c.
>>
>> Fixes: 21083f51521f ("cxl/pmem: Register 'pmem' / cxl_nvdimm devices")
>> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
>> ---
>>  drivers/cxl/core/pmem.c | 13 +++++++++++++
>>  drivers/cxl/pmem.c      | 13 +++++++++++++
>>  2 files changed, 26 insertions(+)
>>
>> diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c
>> index 50557d71a45f..f7f784047450 100644
>> --- a/drivers/cxl/core/pmem.c
>> +++ b/drivers/cxl/core/pmem.c
>> @@ -175,6 +175,19 @@ int devm_cxl_add_nvdimm(struct device *host, struct cxl_port *port,
>>  	if (!cxl_nvb)
>>  		return -ENODEV;
>>  
>> +	/*
>> +	 * Take the uport_dev lock to guard against race of nvdimm_bus object.
>> +	 * cxl_acpi_probe() registers the nvdimm_bus and is done under the
>> +	 * root port uport_dev lock.
>> +	 *
>> +	 * Take the cxl_nvb device lock to ensure that cxl_nvb driver is in a
>> +	 * consistent state. And the driver registers nvdimm_bus.
>> +	 */
>> +	guard(device)(cxl_nvb->port->uport_dev);
>> +	guard(device)(&cxl_nvb->dev);
>> +	if (!cxl_nvb->nvdimm_bus)
>> +		return -ENODEV;
>> +
>>  	cxl_nvd = cxl_nvdimm_alloc(cxl_nvb, cxlmd);
>>  	if (IS_ERR(cxl_nvd)) {
>>  		rc = PTR_ERR(cxl_nvd);
>> diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
>> index 6f010b3cce44..0b200be1fd36 100644
>> --- a/drivers/cxl/pmem.c
>> +++ b/drivers/cxl/pmem.c
>> @@ -85,6 +85,15 @@ struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
>>  	if (rc)
>>  		goto err;
>>  
>> +	/* Ensure that cxl_nvb driver has been bound since it is synchronous. */
>> +	device_lock(dev);
>> +	if (!dev->driver) {
>> +		device_unlock(dev);
>> +		unregister_nvb(cxl_nvb);
>> +		return ERR_PTR(-ENODEV);
>> +	}
>> +	device_unlock(dev);
> 
> This can be a cxl_nvdimm_bridge_failed_attach() helper what uses
> guard().
> 
>> +
>>  	rc = devm_add_action_or_reset(host, unregister_nvb, cxl_nvb);
>>  	if (rc)
>>  		return ERR_PTR(rc);
>> @@ -213,6 +222,9 @@ static int cxl_nvdimm_probe(struct device *dev)
>>  	struct nvdimm *nvdimm;
>>  	int rc;
>>  
>> +	if (!cxl_nvb->nvdimm_bus)
>> +		return -ENODEV;
> 
> If I understand the failure scenario correctly, this is racy. What keeps
> nvdimm_bus valid for the duration of cxl_nvdimm probe?

With patch 3 merged into this one, we can drop this check as there wouldn't be any devices to probe.

> 
> This is why I think the cxl_nvdimm_bridge disable path needs to evacuate
> all prior cxl_nvdimm objects that saw nvdimm_bus live at their
> registration time.

Right I split that out because I was worried about patch size for review. I'll squash the two.

[PATCH v2 3/3] cxl: Clean up cxl_nvdimm devices when removing nvdimm_bus

[Dave Jiang]

In the current implementation, when the nvdimm_bus is removed, the
cxl_nvdimm devices only have the driver unbound which leaves orphaned
cxl_nvdimm devices around. Add removal of cxl_nvdimm devices when the
nvdimm_bus object is removed on cxl_nvb driver removal.

Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
 drivers/cxl/core/pmem.c | 6 +++++-
 drivers/cxl/cxl.h       | 1 +
 drivers/cxl/pmem.c      | 5 +++++
 3 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c
index f7f784047450..4edc87ceb993 100644
--- a/drivers/cxl/core/pmem.c
+++ b/drivers/cxl/core/pmem.c
@@ -142,18 +142,22 @@ static struct cxl_nvdimm *cxl_nvdimm_alloc(struct cxl_nvdimm_bridge *cxl_nvb,
 	return cxl_nvd;
 }
 
-static void cxlmd_release_nvdimm(void *_cxlmd)
+void cxlmd_release_nvdimm(void *_cxlmd)
 {
 	struct cxl_memdev *cxlmd = _cxlmd;
 	struct cxl_nvdimm *cxl_nvd = cxlmd->cxl_nvd;
 	struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
 
+	dev_info(&cxlmd->dev, "XXX release nvdimm %s\n",
+		 cxl_nvd ? dev_name(&cxl_nvd->dev) : "none");
+
 	cxl_nvd->cxlmd = NULL;
 	cxlmd->cxl_nvd = NULL;
 	cxlmd->cxl_nvb = NULL;
 	device_unregister(&cxl_nvd->dev);
 	put_device(&cxl_nvb->dev);
 }
+EXPORT_SYMBOL_NS_GPL(cxlmd_release_nvdimm, "CXL");
 
 /**
  * devm_cxl_add_nvdimm() - add a bridge between a cxl_memdev and an nvdimm
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 3b79d5c4765a..3f0d927e5772 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -926,6 +926,7 @@ bool is_cxl_nvdimm(struct device *dev);
 int devm_cxl_add_nvdimm(struct device *host, struct cxl_port *port,
 			struct cxl_memdev *cxlmd);
 struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_port *port);
+void cxlmd_release_nvdimm(void *cxlmd);
 
 #ifdef CONFIG_CXL_REGION
 bool is_cxl_pmem_region(struct device *dev);
diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
index 0b200be1fd36..edce5269a682 100644
--- a/drivers/cxl/pmem.c
+++ b/drivers/cxl/pmem.c
@@ -397,6 +397,7 @@ static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
 static int detach_nvdimm(struct device *dev, void *data)
 {
 	struct cxl_nvdimm *cxl_nvd;
+	struct cxl_memdev *cxlmd;
 	bool release = false;
 
 	if (!is_cxl_nvdimm(dev))
@@ -411,6 +412,10 @@ static int detach_nvdimm(struct device *dev, void *data)
 	}
 	if (release)
 		device_release_driver(dev);
+
+	cxlmd = cxl_nvd->cxlmd;
+	devm_release_action(&cxlmd->dev, cxlmd_release_nvdimm, cxlmd);
+
 	return 0;
 }
 
-- 
2.52.0

Re: [PATCH v2 3/3] cxl: Clean up cxl_nvdimm devices when removing nvdimm_bus

[dan.j.williams]

Dave Jiang wrote:
> In the current implementation, when the nvdimm_bus is removed, the
> cxl_nvdimm devices only have the driver unbound which leaves orphaned
> cxl_nvdimm devices around. Add removal of cxl_nvdimm devices when the
> nvdimm_bus object is removed on cxl_nvb driver removal.
> 
> Signed-off-by: Dave Jiang <dave.jiang@intel.com>
> ---
>  drivers/cxl/core/pmem.c | 6 +++++-
>  drivers/cxl/cxl.h       | 1 +
>  drivers/cxl/pmem.c      | 5 +++++
>  3 files changed, 11 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c
> index f7f784047450..4edc87ceb993 100644
> --- a/drivers/cxl/core/pmem.c
> +++ b/drivers/cxl/core/pmem.c
> @@ -142,18 +142,22 @@ static struct cxl_nvdimm *cxl_nvdimm_alloc(struct cxl_nvdimm_bridge *cxl_nvb,
>  	return cxl_nvd;
>  }
>  
> -static void cxlmd_release_nvdimm(void *_cxlmd)
> +void cxlmd_release_nvdimm(void *_cxlmd)
>  {
>  	struct cxl_memdev *cxlmd = _cxlmd;
>  	struct cxl_nvdimm *cxl_nvd = cxlmd->cxl_nvd;
>  	struct cxl_nvdimm_bridge *cxl_nvb = cxlmd->cxl_nvb;
>  
> +	dev_info(&cxlmd->dev, "XXX release nvdimm %s\n",
> +		 cxl_nvd ? dev_name(&cxl_nvd->dev) : "none");
> +

Stale debug?

>  	cxl_nvd->cxlmd = NULL;
>  	cxlmd->cxl_nvd = NULL;
>  	cxlmd->cxl_nvb = NULL;
>  	device_unregister(&cxl_nvd->dev);
>  	put_device(&cxl_nvb->dev);
>  }
> +EXPORT_SYMBOL_NS_GPL(cxlmd_release_nvdimm, "CXL");
>  
>  /**
>   * devm_cxl_add_nvdimm() - add a bridge between a cxl_memdev and an nvdimm
> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
> index 3b79d5c4765a..3f0d927e5772 100644
> --- a/drivers/cxl/cxl.h
> +++ b/drivers/cxl/cxl.h
> @@ -926,6 +926,7 @@ bool is_cxl_nvdimm(struct device *dev);
>  int devm_cxl_add_nvdimm(struct device *host, struct cxl_port *port,
>  			struct cxl_memdev *cxlmd);
>  struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_port *port);
> +void cxlmd_release_nvdimm(void *cxlmd);
>  
>  #ifdef CONFIG_CXL_REGION
>  bool is_cxl_pmem_region(struct device *dev);
> diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c
> index 0b200be1fd36..edce5269a682 100644
> --- a/drivers/cxl/pmem.c
> +++ b/drivers/cxl/pmem.c
> @@ -397,6 +397,7 @@ static int cxl_pmem_ctl(struct nvdimm_bus_descriptor *nd_desc,
>  static int detach_nvdimm(struct device *dev, void *data)
>  {
>  	struct cxl_nvdimm *cxl_nvd;
> +	struct cxl_memdev *cxlmd;
>  	bool release = false;
>  
>  	if (!is_cxl_nvdimm(dev))
> @@ -411,6 +412,10 @@ static int detach_nvdimm(struct device *dev, void *data)
>  	}
>  	if (release)
>  		device_release_driver(dev);
> +
> +	cxlmd = cxl_nvd->cxlmd;
> +	devm_release_action(&cxlmd->dev, cxlmd_release_nvdimm, cxlmd);

You need to hold the cxlmd device lock and check against dev.driver
before invoking its release actions. I worry that lockdep will not like
that, but worth a shot.

It might be sufficient to just set an INVALIDATED flag in the cxl_nvdimm
object that cxl_nvdimm_probe() can check to give up early on trying to
re-enable the device.

This would be similar to this check in cxl_mem_probe():

        if (work_pending(&cxlmd->detach_work))
                return -EBUSY;

...which prevents reattach while detach cleanup is pending.

Re: [PATCH v2 3/3] cxl: Clean up cxl_nvdimm devices when removing nvdimm_bus

[kernel test robot]

Hi Dave,

kernel test robot noticed the following build warnings:

[auto build test WARNING on 63fbf275fa9f18f7020fb8acf54fa107e51d0f23]

url:    https://github.com/intel-lab-lkp/linux/commits/Dave-Jiang/cxl-Move-devm_cxl_add_nvdimm_bridge-to-cxl_pmem-ko/20260212-073008
base:   63fbf275fa9f18f7020fb8acf54fa107e51d0f23
patch link:    https://lore.kernel.org/r/20260211232630.480509-4-dave.jiang%40intel.com
patch subject: [PATCH v2 3/3] cxl: Clean up cxl_nvdimm devices when removing nvdimm_bus
config: s390-allmodconfig (https://download.01.org/0day-ci/archive/20260212/202602121343.8IGnsS7f-lkp@intel.com/config)
compiler: clang version 18.1.8 (https://github.com/llvm/llvm-project 3b5b5c1ec4a3095ab096dd780e84d7ab81f3d7ff)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260212/202602121343.8IGnsS7f-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202602121343.8IGnsS7f-lkp@intel.com/

All warnings (new ones prefixed by >>):

>> drivers/cxl/pmem.c:407:7: warning: variable 'cxl_nvd' is used uninitialized whenever 'if' condition is false [-Wsometimes-uninitialized]
     407 |                 if (dev->driver) {
         |                     ^~~~~~~~~~~
   drivers/cxl/pmem.c:416:10: note: uninitialized use occurs here
     416 |         cxlmd = cxl_nvd->cxlmd;
         |                 ^~~~~~~
   drivers/cxl/pmem.c:407:3: note: remove the 'if' if its condition is always true
     407 |                 if (dev->driver) {
         |                 ^~~~~~~~~~~~~~~~
   drivers/cxl/pmem.c:399:28: note: initialize the variable 'cxl_nvd' to silence this warning
     399 |         struct cxl_nvdimm *cxl_nvd;
         |                                   ^
         |                                    = NULL
   1 warning generated.


vim +407 drivers/cxl/pmem.c

8fdcb1704f61a8 Dan Williams 2021-06-15  396  
19398821b25a9c Dan Williams 2023-01-20  397  static int detach_nvdimm(struct device *dev, void *data)
19398821b25a9c Dan Williams 2023-01-20  398  {
19398821b25a9c Dan Williams 2023-01-20  399  	struct cxl_nvdimm *cxl_nvd;
5f761998c93e30 Dave Jiang   2026-02-11  400  	struct cxl_memdev *cxlmd;
19398821b25a9c Dan Williams 2023-01-20  401  	bool release = false;
19398821b25a9c Dan Williams 2023-01-20  402  
19398821b25a9c Dan Williams 2023-01-20  403  	if (!is_cxl_nvdimm(dev))
19398821b25a9c Dan Williams 2023-01-20  404  		return 0;
19398821b25a9c Dan Williams 2023-01-20  405  
7f569e917b7866 Li Ming      2024-08-30  406  	scoped_guard(device, dev) {
7f569e917b7866 Li Ming      2024-08-30 @407  		if (dev->driver) {
19398821b25a9c Dan Williams 2023-01-20  408  			cxl_nvd = to_cxl_nvdimm(dev);
19398821b25a9c Dan Williams 2023-01-20  409  			if (cxl_nvd->cxlmd && cxl_nvd->cxlmd->cxl_nvb == data)
19398821b25a9c Dan Williams 2023-01-20  410  				release = true;
7f569e917b7866 Li Ming      2024-08-30  411  		}
7f569e917b7866 Li Ming      2024-08-30  412  	}
19398821b25a9c Dan Williams 2023-01-20  413  	if (release)
19398821b25a9c Dan Williams 2023-01-20  414  		device_release_driver(dev);
5f761998c93e30 Dave Jiang   2026-02-11  415  
5f761998c93e30 Dave Jiang   2026-02-11  416  	cxlmd = cxl_nvd->cxlmd;
5f761998c93e30 Dave Jiang   2026-02-11  417  	devm_release_action(&cxlmd->dev, cxlmd_release_nvdimm, cxlmd);
5f761998c93e30 Dave Jiang   2026-02-11  418  
19398821b25a9c Dan Williams 2023-01-20  419  	return 0;
19398821b25a9c Dan Williams 2023-01-20  420  }
19398821b25a9c Dan Williams 2023-01-20  421  

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki