[RFC PATCH 06/18] cxl/port, mem: Make adding an endpoint device type agnostic

[Ben Cheatham <Benjamin.Cheatham@amd.com>]

Change devm_cxl_enumerate_ports() and devm_cxl_add_endpoint() to take a
generic struct device * instead of struct cxl_memdev in preparation of
adding cache devices to the port hierarchy.

Signed-off-by: Ben Cheatham <Benjamin.Cheatham@amd.com>
---
 drivers/cxl/core/port.c | 112 +++++++++++++++++++++++++++-------------
 drivers/cxl/cxl.h       |   4 +-
 drivers/cxl/mem.c       |   8 +--
 drivers/cxl/port.c      |  11 ++--
 drivers/cxl/private.h   |   2 +-
 5 files changed, 90 insertions(+), 47 deletions(-)

diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 0eb406d4a131..5144ddac7145 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -1451,10 +1451,21 @@ static struct device *endpoint_host(struct cxl_port *endpoint)
 	return &port->dev;
 }
 
+static struct cxl_port *cxl_dev_get_endpoint(struct device *cxldev)
+{
+	if (is_cxl_memdev(cxldev))
+		return to_cxl_memdev(cxldev)->endpoint;
+
+	if (is_cxl_cachedev(cxldev))
+		return to_cxl_cachedev(cxldev)->endpoint;
+
+	return NULL;
+}
+
 static void delete_endpoint(void *data)
 {
-	struct cxl_memdev *cxlmd = data;
-	struct cxl_port *endpoint = cxlmd->endpoint;
+	struct device *cxldev = data;
+	struct cxl_port *endpoint = cxl_dev_get_endpoint(cxldev);
 	struct device *host = endpoint_host(endpoint);
 
 	scoped_guard(device, host) {
@@ -1463,21 +1474,34 @@ static void delete_endpoint(void *data)
 			devm_release_action(host, cxl_unlink_uport, endpoint);
 			devm_release_action(host, unregister_port, endpoint);
 		}
-		cxlmd->endpoint = NULL;
+
+		if (is_cxl_memdev(cxldev))
+			to_cxl_memdev(cxldev)->endpoint = NULL;
+
+		if (is_cxl_cachedev(cxldev))
+			to_cxl_cachedev(cxldev)->endpoint = NULL;
 	}
 	put_device(&endpoint->dev);
 	put_device(host);
 }
 
-int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
+static void cxl_dev_set_depth(struct device *cxldev, int depth)
+{
+	if (is_cxl_memdev(cxldev))
+		to_cxl_memdev(cxldev)->depth = depth;
+
+	if (is_cxl_cachedev(cxldev))
+		to_cxl_cachedev(cxldev)->depth = depth;
+}
+
+int cxl_endpoint_autoremove(struct device *cxldev, struct cxl_port *endpoint)
 {
 	struct device *host = endpoint_host(endpoint);
-	struct device *dev = &cxlmd->dev;
 
 	get_device(host);
 	get_device(&endpoint->dev);
-	cxlmd->depth = endpoint->depth;
-	return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
+	cxl_dev_set_depth(cxldev, endpoint->depth);
+	return devm_add_action_or_reset(cxldev, delete_endpoint, cxldev);
 }
 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, "CXL");
 
@@ -1711,7 +1735,7 @@ static struct cxl_dport *devm_cxl_add_dport_by_uport(struct device *uport_dev,
 	return devm_cxl_port_add_dport(port, dport_dev);
 }
 
-static int add_port_attach_ep(struct cxl_memdev *cxlmd,
+static int add_port_attach_ep(struct device *cxldev,
 			      struct device *uport_dev,
 			      struct device *dport_dev)
 {
@@ -1726,7 +1750,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
 		 * CXL-root 'cxl_port' on a previous iteration, fail for now to
 		 * be re-probed after platform driver attaches.
 		 */
-		dev_dbg(&cxlmd->dev, "%s is a root dport\n",
+		dev_dbg(cxldev, "%s is a root dport\n",
 			dev_name(dport_dev));
 		return -ENXIO;
 	}
@@ -1747,9 +1771,10 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
 		struct cxl_dport *new_dport;
 
 		if (!parent_port->dev.driver) {
-			dev_warn(&cxlmd->dev,
-				 "port %s:%s disabled, failed to enumerate CXL.mem\n",
-				 dev_name(&parent_port->dev), dev_name(uport_dev));
+			dev_warn(cxldev,
+				 "port %s:%s disabled, failed to enumerate CXL.%s\n",
+				 dev_name(&parent_port->dev), dev_name(uport_dev),
+				 is_cxl_memdev(cxldev) ? "mem" : "cache");
 			return -ENXIO;
 		}
 
@@ -1786,9 +1811,9 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
 		dport = new_dport;
 	}
 
-	dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
+	dev_dbg(cxldev, "add to new port %s:%s\n",
 		dev_name(&port->dev), dev_name(port->uport_dev));
-	rc = cxl_add_ep(dport, &cxlmd->dev);
+	rc = cxl_add_ep(dport, cxldev);
 	if (rc == -EBUSY) {
 		/*
 		 * "can't" happen, but this error code means
@@ -1802,14 +1827,13 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
 
 #define CXL_ITER_LEVEL_SWITCH	1
 
-static int get_hostbridge_port_devices(struct cxl_memdev *cxlmd,
+static int get_hostbridge_port_devices(struct device *cxldev,
 				       struct device **hb_uport_dev,
 				       struct device **hb_dport_dev)
 {
-	struct device *dev = &cxlmd->dev;
 	struct device *iter;
 
-	for (iter = dev; iter; iter = grandparent(iter)) {
+	for (iter = cxldev; iter; iter = grandparent(iter)) {
 		struct device *dport_dev = grandparent(iter);
 		struct device *uport_dev = dport_dev->parent;
 
@@ -1823,13 +1847,13 @@ static int get_hostbridge_port_devices(struct cxl_memdev *cxlmd,
 	return -ENODEV;
 }
 
-static int cxl_hostbridge_port_setup(struct cxl_memdev *cxlmd)
+static int cxl_hostbridge_port_setup(struct device *cxldev)
 {
 	struct device *hb_uport_dev, *hb_dport_dev;
 	struct cxl_dport *dport = NULL;
 	int rc;
 
-	rc = get_hostbridge_port_devices(cxlmd, &hb_uport_dev, &hb_dport_dev);
+	rc = get_hostbridge_port_devices(cxldev, &hb_uport_dev, &hb_dport_dev);
 	if (rc)
 		return -ENODEV;
 
@@ -1852,7 +1876,7 @@ static int cxl_hostbridge_port_setup(struct cxl_memdev *cxlmd)
 		guard(device)(&port->dev);
 		dport = devm_cxl_port_add_dport(port, hb_dport_dev);
 		if (IS_ERR(dport) && PTR_ERR(dport) != -EEXIST) {
-			dev_dbg(&cxlmd->dev,
+			dev_dbg(cxldev,
 				"failed to add dport %s to port %s: %ld\n",
 				dev_name(hb_dport_dev), dev_name(&port->dev),
 				PTR_ERR(dport));
@@ -1872,7 +1896,7 @@ static int cxl_hostbridge_port_setup(struct cxl_memdev *cxlmd)
 	/* Add the dport that goes with the newly created port */
 	dport = devm_cxl_add_dport_by_uport(hb_uport_dev, hb_dport_dev);
 	if (IS_ERR(dport) && PTR_ERR(dport) != -EEXIST) {
-		dev_dbg(&cxlmd->dev,
+		dev_dbg(cxldev,
 			"failed to add dport %s to port %s: %ld\n",
 			dev_name(hb_dport_dev), dev_name(&cxl_root->port.dev),
 			PTR_ERR(dport));
@@ -1882,27 +1906,45 @@ static int cxl_hostbridge_port_setup(struct cxl_memdev *cxlmd)
 	return 0;
 }
 
-int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
+static struct cxl_dev_state *cxl_get_dev_state(struct device *dev)
 {
-	struct device *dev = &cxlmd->dev;
+	if (is_cxl_memdev(dev))
+		return to_cxl_memdev(dev)->cxlds;
+
+	if (is_cxl_cachedev(dev))
+		return to_cxl_cachedev(dev)->cxlds;
+
+	return ERR_PTR(-EINVAL);
+}
+
+int devm_cxl_enumerate_ports(struct device *cxldev)
+{
+	struct cxl_dev_state *cxlds = cxl_get_dev_state(cxldev);
 	struct device *dgparent;
 	struct device *iter;
 	int rc, i;
 
+	/* An error here means @cxldev isn't a supported device type */
+	if (IS_ERR(cxlds))
+		return PTR_ERR(cxlds);
+
 	/*
 	 * Skip intermediate port enumeration in the RCH case, there
 	 * are no ports in between a host bridge and an endpoint.
 	 */
-	if (cxlmd->cxlds->rcd)
+	if (cxlds->rcd)
 		return 0;
 
-	rc = cxl_hostbridge_port_setup(cxlmd);
+	rc = cxl_hostbridge_port_setup(cxldev);
 	if (rc)
 		return rc;
 
-	rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
-	if (rc)
-		return rc;
+	if (is_cxl_memdev(cxldev)) {
+		rc = devm_add_action_or_reset(cxldev, cxl_detach_ep,
+					      to_cxl_memdev(cxldev));
+		if (rc)
+			return rc;
+	}
 
 	/*
 	 * Scan for and add all cxl_ports in this device's ancestry.
@@ -1910,7 +1952,7 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
 	 * attempt fails.
 	 */
 retry:
-	for (i = 0, iter = dev; iter; i++, iter = grandparent(iter)) {
+	for (i = 0, iter = cxldev; iter; i++, iter = grandparent(iter)) {
 		struct device *dport_dev = grandparent(iter);
 		struct device *uport_dev;
 		struct cxl_dport *dport;
@@ -1920,22 +1962,22 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
 
 		uport_dev = dport_dev->parent;
 		if (!uport_dev) {
-			dev_warn(dev, "at %s no parent for dport: %s\n",
+			dev_warn(cxldev, "at %s no parent for dport: %s\n",
 				 dev_name(iter), dev_name(dport_dev));
 			return -ENXIO;
 		}
 
-		dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
+		dev_dbg(cxldev, "scan: iter: %s dport_dev: %s parent: %s\n",
 			dev_name(iter), dev_name(dport_dev),
 			dev_name(uport_dev));
 		struct cxl_port *port __free(put_cxl_port) =
 			find_cxl_port(dport_dev, &dport);
 		if (port) {
-			dev_dbg(&cxlmd->dev,
+			dev_dbg(cxldev,
 				"found already registered port %s:%s\n",
 				dev_name(&port->dev),
 				dev_name(port->uport_dev));
-			rc = cxl_add_ep(dport, &cxlmd->dev);
+			rc = cxl_add_ep(dport, cxldev);
 
 			/*
 			 * If the endpoint already exists in the port's list,
@@ -1960,7 +2002,7 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
 			 */
 			if (i >= CXL_ITER_LEVEL_SWITCH) {
 				struct cxl_port *pport __free(put_cxl_port) =
-					cxl_dev_find_port(&cxlmd->dev, &dport);
+					cxl_dev_find_port(cxldev, &dport);
 				if (!pport)
 					goto retry;
 			}
@@ -1979,7 +2021,7 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
 			if (IS_ERR(dport))
 				return PTR_ERR(dport);
 		} else {
-			rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
+			rc = add_port_attach_ep(cxldev, uport_dev, dport_dev);
 			/* port missing, try to add parent */
 			if (rc == -EAGAIN)
 				continue;
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 3ad25229a48b..c857c112772f 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -869,7 +869,7 @@ DEFINE_FREE(put_cxl_port, struct cxl_port *, if (!IS_ERR_OR_NULL(_T)) put_device
 DEFINE_FREE(put_cxl_root_decoder, struct cxl_root_decoder *, if (!IS_ERR_OR_NULL(_T)) put_device(&_T->cxlsd.cxld.dev))
 DEFINE_FREE(put_cxl_region, struct cxl_region *, if (!IS_ERR_OR_NULL(_T)) put_device(&_T->dev))
 
-int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd);
+int devm_cxl_enumerate_ports(struct device *cxldev);
 void cxl_bus_rescan(void);
 void cxl_bus_drain(void);
 struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
@@ -913,7 +913,7 @@ static inline int cxl_root_decoder_autoremove(struct device *host,
 {
 	return cxl_decoder_autoremove(host, &cxlrd->cxlsd.cxld);
 }
-int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);
+int cxl_endpoint_autoremove(struct device *cxldev, struct cxl_port *endpoint);
 
 /**
  * struct cxl_endpoint_dvsec_info - Cached DVSEC info
diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c
index cdac7bfa2289..c615e6c4ee60 100644
--- a/drivers/cxl/mem.c
+++ b/drivers/cxl/mem.c
@@ -104,9 +104,11 @@ static int cxl_mem_probe(struct device *dev)
 	if (rc)
 		return rc;
 
-	rc = devm_cxl_enumerate_ports(cxlmd);
-	if (rc)
+	rc = devm_cxl_enumerate_ports(&cxlmd->dev);
+	if (rc) {
+		cxlmd->endpoint = ERR_PTR(rc);
 		return rc;
+	}
 
 	struct cxl_port *parent_port __free(put_cxl_port) =
 		cxl_dev_find_port(&cxlmd->dev, &dport);
@@ -138,7 +140,7 @@ static int cxl_mem_probe(struct device *dev)
 			return -ENXIO;
 		}
 
-		rc = devm_cxl_add_endpoint(endpoint_parent, cxlmd, dport);
+		rc = devm_cxl_add_endpoint(endpoint_parent, &cxlmd->dev, dport);
 		if (rc)
 			return rc;
 	}
diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c
index 7d88a8bec8e8..aa6ee6cc714e 100644
--- a/drivers/cxl/port.c
+++ b/drivers/cxl/port.c
@@ -199,7 +199,7 @@ static struct cxl_driver cxl_port_driver = {
 	},
 };
 
-int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd,
+int devm_cxl_add_endpoint(struct device *host, struct device *cxldev,
 			  struct cxl_dport *parent_dport)
 {
 	struct cxl_port *parent_port = parent_dport->port;
@@ -214,23 +214,22 @@ int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd,
 	     down = iter, iter = to_cxl_port(iter->dev.parent)) {
 		struct cxl_ep *ep;
 
-		ep = cxl_ep_load(iter, &cxlmd->dev);
+		ep = cxl_ep_load(iter, cxldev);
 		ep->next = down;
 	}
 
 	/* Note: endpoint port component registers are derived from @cxlds */
-	endpoint = devm_cxl_add_port(host, &cxlmd->dev, CXL_RESOURCE_NONE,
+	endpoint = devm_cxl_add_port(host, cxldev, CXL_RESOURCE_NONE,
 				     parent_dport);
 	if (IS_ERR(endpoint))
 		return PTR_ERR(endpoint);
 
-	rc = cxl_endpoint_autoremove(cxlmd, endpoint);
+	rc = cxl_endpoint_autoremove(cxldev, endpoint);
 	if (rc)
 		return rc;
 
 	if (!endpoint->dev.driver) {
-		dev_err(&cxlmd->dev, "%s failed probe\n",
-			dev_name(&endpoint->dev));
+		dev_err(cxldev, "%s failed probe\n", dev_name(&endpoint->dev));
 		return -ENXIO;
 	}
 
diff --git a/drivers/cxl/private.h b/drivers/cxl/private.h
index 49c588b95c19..04ec30ab5f31 100644
--- a/drivers/cxl/private.h
+++ b/drivers/cxl/private.h
@@ -16,6 +16,6 @@ struct cxl_cachedev *cxl_cachedev_alloc(struct cxl_dev_state *cxlds,
 					const struct cxl_dev_ops *ops);
 struct cxl_cachedev *devm_cxl_cachedev_add_or_reset(struct device *host,
 						    struct cxl_cachedev *cxlcd);
-int devm_cxl_add_endpoint(struct device *host, struct cxl_memdev *cxlmd,
+int devm_cxl_add_endpoint(struct device *host, struct device *cxldev,
 			  struct cxl_dport *parent_dport);
 #endif /* __CXL_PRIVATE_H__ */
-- 
2.34.1