[PATCH 09/15] cxl/core: Add Cache ID verification

[Ben Cheatham <Benjamin.Cheatham@amd.com>]

Check if system firmware has pre-programmed a CXL.cache device's cache
id as part of cxl_cache::probe(). A cache id is required when multiple
CXL.cache devices are present under a host bridge, so fail probe if the
id has not been programmed and another device is present.

Also add @hdmd to struct cxl_dev_state for endpoint drivers to indicate
whether their device is using HDM-D flows. This is required for
correctly validating cache id programming.

Signed-off-by: Ben Cheatham <Benjamin.Cheatham@amd.com>
---
 drivers/cxl/cache.c         |  94 ++++++++++++++++++++-
 drivers/cxl/core/cache.c    | 158 ++++++++++++++++++++++++++++++++++++
 drivers/cxl/core/cachedev.c |   1 +
 drivers/cxl/core/port.c     |   1 +
 drivers/cxl/cxl.h           |  16 ++++
 drivers/cxl/cxlcache.h      |   6 ++
 include/cxl/cxl.h           |   4 +
 7 files changed, 278 insertions(+), 2 deletions(-)

diff --git a/drivers/cxl/cache.c b/drivers/cxl/cache.c
index dfefd301696a..6c098010149a 100644
--- a/drivers/cxl/cache.c
+++ b/drivers/cxl/cache.c
@@ -84,6 +84,90 @@ static int cxl_endpoint_cache_enabled(struct cxl_port *endpoint)
 	return 0;
 }
 
+static struct cxl_port *find_host_bridge(struct cxl_port *endpoint)
+{
+	struct cxl_port *parent = endpoint->parent_dport->port;
+	struct cxl_port *hb = endpoint;
+
+	if (is_cxl_root(parent))
+		return NULL;
+
+	while (!is_cxl_root(parent)) {
+		hb = parent;
+		parent = hb->parent_dport->port;
+	}
+
+	return hb;
+}
+
+static int get_num_cachedevs_present(struct cxl_port *host_bridge, u32 *num)
+{
+	u32 num_cachedevs = 0;
+	unsigned long index;
+	struct cxl_ep *ep;
+
+	xa_for_each(&host_bridge->endpoints, index, ep) {
+		if (is_cxl_cachedev(ep->ep))
+			num_cachedevs++;
+	}
+
+	*num = num_cachedevs;
+	return 0;
+}
+
+static void deprogram_cache_id(void *_cxlcd)
+{
+	struct cxl_cachedev *cxlcd = _cxlcd;
+	struct cxl_port *hb = find_host_bridge(cxlcd->endpoint);
+
+	if (cxlcd->cache_id == CXL_CACHE_ID_NO_ID)
+		return;
+
+	guard(device)(&hb->dev);
+	cxl_free_cache_id(cxlcd);
+}
+
+static int program_cache_id(struct cxl_cachedev *cxlcd)
+{
+	struct cxl_port *endpoint = cxlcd->endpoint;
+	struct cxl_port *hb = find_host_bridge(endpoint);
+	struct device *dev = &cxlcd->dev;
+	u32 num_cachedevs;
+	int rc;
+
+	if (!hb)
+		return -ENODEV;
+
+	/*
+	 * Cache ids are unique to the host bridge, so synchronize programming
+	 * around the bridge's device lock
+	 */
+
+	guard(device)(&hb->dev);
+	rc = get_num_cachedevs_present(hb, &num_cachedevs);
+	if (rc)
+		return rc;
+
+	if (!cxl_cache_id_supported(cxlcd))
+		return num_cachedevs > 1 ? -ENXIO : 0;
+
+	rc = cxl_cachedev_validate_cache_id(cxlcd);
+	if (rc && num_cachedevs > 1) {
+		dev_err(dev,
+			"Cache id not programmed with other CXL.cache devices present: %d\n",
+			rc);
+		return rc;
+	} else if (rc) {
+		cxlcd->cache_id = 0;
+	}
+
+	rc = cxl_allocate_cache_id(cxlcd);
+	if (rc)
+		return rc;
+
+	return 0;
+}
+
 /**
  * devm_cxl_add_cachedev - Add a CXL cache device
  * @cxlds: CXL device state to associate with the cachedev
@@ -142,10 +226,16 @@ static int cxl_cache_probe(struct device *dev)
 	}
 
 	rc = cxl_endpoint_cache_enabled(cxlcd->endpoint);
-	if (rc)
+	if (rc) {
 		dev_err(dev, "CXL.cache not enabled on parent port(s)");
+		return rc;
+	}
+
+	rc = program_cache_id(cxlcd);
+	if (rc)
+		return rc;
 
-	return rc;
+	return devm_add_action_or_reset(dev, deprogram_cache_id, cxlcd);
 }
 
 static struct cxl_driver cxl_cache_driver = {
diff --git a/drivers/cxl/core/cache.c b/drivers/cxl/core/cache.c
index 17df0e603b68..be570d3dc609 100644
--- a/drivers/cxl/core/cache.c
+++ b/drivers/cxl/core/cache.c
@@ -39,3 +39,161 @@ int cxl_dport_map_cache_id_dc(struct cxl_dport *dport)
 	return cxl_map_component_regs(map, &dport->regs.component,
 				      BIT(CXL_CM_CAP_CAP_ID_CACHE_ID_DC));
 }
+
+static int cache_decoder_committed(struct cxl_dport *dport)
+{
+	u32 cap, stat;
+
+	cap = readl(dport->regs.ciddc + CXL_CACHE_ID_DC_CAP_OFFSET);
+	if (!FIELD_GET(CXL_CACHE_ID_DC_CAP_COMMIT_REQ, cap))
+		return -ENXIO;
+
+	stat = readl(dport->regs.ciddc + CXL_CACHE_ID_DC_STATUS_OFFSET);
+	return FIELD_GET(CXL_CACHE_ID_DC_STATUS_COMMITTED, stat);
+}
+
+static int cache_decoder_get_id(struct cxl_dport *dport)
+{
+	u32 ctrl;
+
+	ctrl = readl(dport->regs.ciddc + CXL_CACHE_ID_DC_CTRL_OFFSET);
+	if (FIELD_GET(CXL_CACHE_ID_DC_CTRL_HDMD_PRESENT, ctrl))
+		 return FIELD_GET(CXL_CACHE_ID_DC_CTRL_HDMD_ID, ctrl);
+
+	return FIELD_GET(CXL_CACHE_ID_DC_CTRL_LOCAL_ID, ctrl);
+}
+
+static bool cache_decoder_valid(struct cxl_dport *dport, int id, bool endpoint)
+{
+	struct pci_dev *pdev = to_pci_dev(dport->dport_dev);
+	bool flit_256b;
+	u32 ctrl;
+
+	if (!dev_is_pci(dport->dport_dev))
+		return false;
+
+	flit_256b = cxl_pci_flit_256(pdev);
+	if (id && !flit_256b)
+		return false;
+
+	ctrl = readl(dport->regs.ciddc + CXL_CACHE_ID_DC_CTRL_OFFSET);
+	if (endpoint || !flit_256b) {
+		if (!FIELD_GET(CXL_CACHE_ID_DC_CTRL_ASGN_ID, ctrl))
+			return false;
+	} else if (!FIELD_GET(CXL_CACHE_ID_DC_CTRL_FWD_ID, ctrl)) {
+		return false;
+	}
+
+	return cache_decoder_get_id(dport) == id;
+}
+
+static int cache_idrt_committed(struct cxl_port *port)
+{
+	u32 cap, stat;
+
+	cap = readl(port->regs.cidrt + CXL_CACHE_ID_RT_CAP_OFFSET);
+	if (!FIELD_GET(CXL_CACHE_ID_RT_CAP_COMMIT_REQ, cap))
+		return true;
+
+	stat = readl(port->regs.cidrt + CXL_CACHE_ID_RT_STATUS_OFFSET);
+	return FIELD_GET(CXL_CACHE_ID_RT_STATUS_COMMITTED, stat);
+}
+
+static int cache_idrt_entry_valid(struct cxl_port *port, int id)
+{
+	u16 entry;
+	u32 cap;
+
+	cap = readl(port->regs.cidrt + CXL_CACHE_ID_RT_CAP_OFFSET);
+	if (id >= FIELD_GET(CXL_CACHE_ID_RT_CAP_TARGET_CNT, cap))
+		return false;
+
+	entry = readw(port->regs.cidrt + CXL_CACHE_ID_RT_TARGETN_OFFSET(id));
+	return FIELD_GET(CXL_CACHE_ID_RT_TARGETN_VALID, entry);
+}
+
+static struct ida *find_cache_id_ida(struct cxl_port *port)
+{
+	struct cxl_port *parent = parent_port_of(port);
+
+	while (parent) {
+		if (is_cxl_root(parent))
+			return &port->cache_ida;
+
+		port = parent;
+		parent = parent_port_of(port);
+	}
+
+	return NULL;
+}
+
+int cxl_allocate_cache_id(struct cxl_cachedev *cxlcd)
+{
+	struct ida *ida = find_cache_id_ida(cxlcd->endpoint);
+	int id;
+
+	if (!ida)
+		return -ENOSPC;
+
+	id = ida_alloc_range(ida, cxlcd->cache_id, cxlcd->cache_id, GFP_KERNEL);
+	if (id != cxlcd->cache_id)
+		return -EINVAL;
+
+	return 0;
+}
+EXPORT_SYMBOL_FOR_MODULES(cxl_allocate_cache_id, "cxl_cache");
+
+void cxl_free_cache_id(struct cxl_cachedev *cxlcd)
+{
+	struct ida *ida = find_cache_id_ida(cxlcd->endpoint);
+
+	if (ida)
+		ida_free(ida, cxlcd->cache_id);
+}
+EXPORT_SYMBOL_FOR_MODULES(cxl_free_cache_id, "cxl_cache");
+
+bool cxl_cache_id_supported(struct cxl_cachedev *cxlcd)
+{
+	struct cxl_dport *dport = cxlcd->endpoint->parent_dport;
+	struct cxl_port *port = dport->port;
+
+	for (; !is_cxl_root(port);
+	     dport = port->parent_dport, port = dport->port) {
+		if (!port->regs.cidrt || !dport->regs.ciddc)
+			return false;
+	}
+
+	return true;
+}
+EXPORT_SYMBOL_FOR_MODULES(cxl_cache_id_supported, "cxl_cache");
+
+int cxl_cachedev_validate_cache_id(struct cxl_cachedev *cxlcd)
+{
+	struct cxl_dport *dport = cxlcd->endpoint->parent_dport;
+	struct cxl_port *port = dport->port;
+	int id = CXL_CACHE_ID_NO_ID;
+	bool endpoint = true;
+
+	cxlcd->cache_id = CXL_CACHE_ID_NO_ID;
+	id = cache_decoder_get_id(dport);
+
+	while (!is_cxl_root(port)) {
+		if (!cache_decoder_valid(dport, id, endpoint) ||
+		    !cache_decoder_committed(dport))
+			return -EINVAL;
+
+		endpoint = false;
+
+		if (!cache_idrt_entry_valid(port, id) ||
+		    !cache_idrt_committed(port))
+			return -EINVAL;
+
+		dport = port->parent_dport;
+		port = dport->port;
+	}
+
+	cxlcd->cache_id = id;
+	return 0;
+}
+EXPORT_SYMBOL_FOR_MODULES(cxl_cachedev_validate_cache_id, "cxl_cache");
+>>>>>>> conflict 1 of 1 ends
diff --git a/drivers/cxl/core/cachedev.c b/drivers/cxl/core/cachedev.c
index 2bd69133c631..38f0adf2b197 100644
--- a/drivers/cxl/core/cachedev.c
+++ b/drivers/cxl/core/cachedev.c
@@ -72,6 +72,7 @@ static struct cxl_cachedev *cxl_cachedev_alloc(struct cxl_dev_state *cxlds)
 	cxlcd->id = rc;
 	cxlcd->depth = -1;
 	cxlcd->endpoint = ERR_PTR(-ENODEV);
+	cxlcd->cache_id = CXL_CACHE_ID_NO_ID;
 
 	dev = &cxlcd->dev;
 	device_initialize(dev);
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 6da11d28bb5e..047801faa020 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -740,6 +740,7 @@ static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
 		dev->parent = uport_dev;
 
 	ida_init(&port->decoder_ida);
+	ida_init(&port->cache_ida);
 	port->hdm_end = -1;
 	port->commit_end = -1;
 	xa_init(&port->dports);
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 9f0c1a39aaeb..ed6b56a269ee 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -229,9 +229,23 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
 /* CXL 4.0 8.2.4.28.1 CXL Cache ID Route Table Capability Structure */
 #define CXL_CACHE_ID_RT_CAP_OFFSET 0x0
 #define   CXL_CACHE_ID_RT_CAP_TARGET_CNT GENMASK(4, 0)
+#define   CXL_CACHE_ID_RT_CAP_COMMIT_REQ BIT(16)
+#define CXL_CACHE_ID_RT_STATUS_OFFSET 0x8
+#define   CXL_CACHE_ID_RT_STATUS_COMMITTED BIT(0)
 #define CXL_CACHE_ID_RT_TARGETN_OFFSET(n) (0x10 + (2 * (n)))
+#define   CXL_CACHE_ID_RT_TARGETN_VALID BIT(0)
 
 /* CXL 4.0 8.2.4.29.1 CXL Cache ID Decoder Capability Structure */
+#define CXL_CACHE_ID_DC_CAP_OFFSET 0x0
+#define   CXL_CACHE_ID_DC_CAP_COMMIT_REQ BIT(0)
+#define CXL_CACHE_ID_DC_CTRL_OFFSET 0x4
+#define   CXL_CACHE_ID_DC_CTRL_FWD_ID BIT(0)
+#define   CXL_CACHE_ID_DC_CTRL_ASGN_ID BIT(1)
+#define   CXL_CACHE_ID_DC_CTRL_HDMD_PRESENT BIT(2)
+#define   CXL_CACHE_ID_DC_CTRL_HDMD_ID GENMASK(11, 8)
+#define   CXL_CACHE_ID_DC_CTRL_LOCAL_ID GENMASK(19, 16)
+#define CXL_CACHE_ID_DC_STATUS_OFFSET 0x8
+#define   CXL_CACHE_ID_DC_STATUS_COMMITTED BIT(0)
 #define CXL_CACHE_ID_DC_CAPABILITY_LENGTH 0xC
 
 void cxl_probe_component_regs(struct device *dev, void __iomem *base,
@@ -578,6 +592,7 @@ struct cxl_dax_region {
  * @cdat_available: Should a CDAT attribute be available in sysfs
  * @pci_latency: Upstream latency in picoseconds
  * @component_reg_phys: Physical address of component register
+ * @cache_ida: ida used for cache id allocations on host bridges
  */
 struct cxl_port {
 	struct device dev;
@@ -603,6 +618,7 @@ struct cxl_port {
 	bool cdat_available;
 	long pci_latency;
 	resource_size_t component_reg_phys;
+	struct ida cache_ida;
 };
 
 struct cxl_root;
diff --git a/drivers/cxl/cxlcache.h b/drivers/cxl/cxlcache.h
index e9a4567b7f2c..1e4a1e61ad7b 100644
--- a/drivers/cxl/cxlcache.h
+++ b/drivers/cxl/cxlcache.h
@@ -11,6 +11,7 @@
  * @cxlds: device state backing this device
  * @detach_work: active cachedev lost a port in its ancestry
  * @endpoint: connection to the CXL port topology for this device
+ * @cache_id: id used for CXL cache routing capabilities
  * @id: id number of this cachedev instance
  * @depth: endpoint port depth in hierarchy
  */
@@ -19,6 +20,7 @@ struct cxl_cachedev {
 	struct cxl_dev_state *cxlds;
 	struct work_struct detach_work;
 	struct cxl_port *endpoint;
+	int cache_id;
 	int id;
 	int depth;
 };
@@ -39,4 +41,8 @@ struct cxl_cachedev *__devm_cxl_add_cachedev(struct cxl_dev_state *cxlds,
 int cxl_accel_read_cache_info(struct cxl_dev_state *cxlds);
 void cxl_clear_cache_enable(struct cxl_dev_state *cxlds);
 int devm_cxl_enable_cache(struct device *host, struct cxl_dev_state *cxlds);
+bool cxl_cache_id_supported(struct cxl_cachedev *cxlcd);
+int cxl_allocate_cache_id(struct cxl_cachedev *cxlcd);
+void cxl_free_cache_id(struct cxl_cachedev *cxlcd);
+int cxl_cachedev_validate_cache_id(struct cxl_cachedev *cxlcd);
 #endif
diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h
index 7a5ed64904a6..b99a9727e284 100644
--- a/include/cxl/cxl.h
+++ b/include/cxl/cxl.h
@@ -155,6 +155,8 @@ struct cxl_dpa_partition {
 
 #define CXL_NR_PARTITIONS_MAX 2
 
+#define CXL_CACHE_ID_NO_ID (-1)
+ 
 /**
  * struct cxl_cache_state - CXL cache device state for use by external drivers
  * @size: Size of device's cache
@@ -182,6 +184,7 @@ struct cxl_cache_state {
  * @cxlmd: The device representing the CXL.mem capabilities of @dev
  * @cxlcd: The device representing the CXL.cache capabilities of @dev
  * @cstate: CXL.cache information for @dev
+ * @hdmd: Whether the device is using HDM-D flows for CXL.cache
  * @reg_map: component and ras register mapping parameters
  * @regs: Parsed register blocks
  * @cxl_dvsec: Offset to the PCIe device DVSEC
@@ -201,6 +204,7 @@ struct cxl_dev_state {
 	struct cxl_memdev *cxlmd;
 	struct cxl_cachedev *cxlcd;
  	struct cxl_cache_state cstate;
+	bool hdmd;
 
 	/* private for Type2 drivers */
 	struct cxl_register_map reg_map;
-- 
2.53.0