[PATCH 12/17] cxl/core: Add CXL snoop filter setup and allocation

[Ben Cheatham <Benjamin.Cheatham@amd.com>]

The CXL snoop filter capability (CXL 3.2 8.2.4.23) is a mandatory
capability for CXL 2.0+ root ports that is used by the host and CXL
devices to manage CXL cache coherence.

Snoop filters may be shared by several CXL root ports, with each snoop
filter having a unique group id. Create snoop filter instances as CXL
root ports are added to the system. If the snoop filter already exists,
add the root port to the existing filter instead.

Snoop filter instances are used to keep track of the amount of CXL cache
space active under the filter. The CXL specification allows rejecting
CXL devices based on the amount of filter subscription, but determining
when the filter is oversaturated is most likely implementation specific.
To address this, add a cxl_core module parameter to specify whether
adding a CXL.cache device's cache to an already full filter should fail
device probe. The default setting of the parameter is to allow
oversaturation of the filter ("strict_snoop=off").

Signed-off-by: Ben Cheatham <Benjamin.Cheatham@amd.com>
---

Jonathan asked about the allocation policy when the snoop filter is
"full". I had the device fail probe beforehand, but it got me thinking
that it may be worthwhile to add an option to over commit the filter
instead, hence the module parameter. I'm game to other suggestions, but
a config symbol didn't feel right since the setting probably varies by
platform/user.

Also I'll add Cache_SF_Coverage support when this series goes out of
RFC (I forgot to do it this time around).

---
 drivers/cxl/cache.c     |  35 ++++++++
 drivers/cxl/core/port.c | 178 ++++++++++++++++++++++++++++++++++++++++
 drivers/cxl/core/regs.c |   7 ++
 drivers/cxl/cxl.h       |  14 ++++
 4 files changed, 234 insertions(+)

diff --git a/drivers/cxl/cache.c b/drivers/cxl/cache.c
index dcc9816d9b37..40c3ffd9c89b 100644
--- a/drivers/cxl/cache.c
+++ b/drivers/cxl/cache.c
@@ -12,6 +12,10 @@
  * device-specific driver is required for discovery and portions of set up.
  */
 
+/* Controls whether failing to allocate snoop filter capacity is tolerated */
+static bool strict_snoop;
+module_param(strict_snoop, bool, 0644);
+
 /**
  * devm_cxl_add_cachedev - Add a CXL cache device
  * @host: devres alloc/release context and parent for the cachedev
@@ -54,6 +58,22 @@ struct cxl_cachedev *devm_cxl_add_cachedev(struct device *host,
 }
 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_cachedev, "CXL");
 
+static int cxl_populate_snoop_gid(struct cxl_cachedev *cxlcd)
+{
+	struct cxl_cache_state *cstate = &cxlcd->cxlds->cstate;
+
+	for (struct cxl_dport *iter = cxlcd->endpoint->parent_dport;
+	     iter && !is_cxl_root(iter->port);
+	     iter = iter->port->parent_dport) {
+		if (iter->snoop_id != CXL_SNOOP_ID_NO_ID) {
+			cstate->snoop_id = iter->snoop_id;
+			return 0;
+		}
+	}
+
+	return -ENXIO;
+}
+
 static int cxl_cache_probe(struct device *dev)
 {
 	struct cxl_cachedev *cxlcd = to_cxl_cachedev(dev);
@@ -93,6 +113,21 @@ static int cxl_cache_probe(struct device *dev)
 			return rc;
 	}
 
+	rc = cxl_populate_snoop_gid(cxlcd);
+	if (rc) {
+		dev_dbg(&cxlcd->dev, "failed to get snoop id: %d\n", rc);
+		return rc;
+	}
+
+	rc = devm_cxl_snoop_filter_alloc_capacity(cxlcd);
+	if (rc) {
+		dev_dbg(&cxlcd->dev,
+			"failed to allocate snoop filter capacity: %d\n", rc);
+
+		if (strict_snoop)
+			return rc;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index dc001e136782..70666a059d1f 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -34,6 +34,8 @@
 static DEFINE_IDA(cxl_port_ida);
 static DEFINE_XARRAY(cxl_root_buses);
 
+static DEFINE_XARRAY(cxl_snoop_filters);
+
 /*
  * The terminal device in PCI is NULL and @platform_bus
  * for platform devices (for cxl_test)
@@ -810,6 +812,175 @@ static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
 	return rc;
 }
 
+struct cxl_snoop_filter {
+	u64 size;
+	struct xarray dports;
+	struct xarray devs;
+	struct mutex lock; /* Used for filter allocations */
+};
+
+static int cxl_snoop_filter_add_dport(struct cxl_dport *dport, u32 gid)
+{
+	struct cxl_snoop_filter *sf;
+
+	sf = xa_load(&cxl_snoop_filters, gid);
+	if (sf)
+		return xa_insert(&sf->dports, (unsigned long)dport->dport_dev,
+				 dport, GFP_KERNEL);
+
+	return -ENXIO;
+}
+
+static struct cxl_snoop_filter *cxl_alloc_snoop_filter(u64 size)
+{
+	struct cxl_snoop_filter *sf;
+
+	sf = kzalloc(sizeof(*sf), GFP_KERNEL);
+	if (!sf)
+		return NULL;
+
+	sf->size = size;
+	xa_init(&sf->dports);
+	xa_init(&sf->devs);
+	mutex_init(&sf->lock);
+
+	return sf;
+}
+
+static void cxl_destroy_snoop_filter(struct cxl_snoop_filter *sf)
+{
+	xa_destroy(&sf->dports);
+	xa_destroy(&sf->devs);
+	mutex_destroy(&sf->lock);
+	kfree(sf);
+}
+
+static int cxl_create_snoop_filter(struct cxl_dport *dport, u32 gid, u64 size)
+{
+	struct cxl_snoop_filter *sf;
+	int rc;
+
+	sf = cxl_alloc_snoop_filter(size);
+	if (!sf)
+		return -ENOMEM;
+
+	rc = xa_insert(&sf->dports, (unsigned long)dport->dport_dev, dport,
+		       GFP_KERNEL);
+	if (rc) {
+		cxl_destroy_snoop_filter(sf);
+		return rc;
+	}
+
+	rc = xa_insert(&cxl_snoop_filters, gid, sf, GFP_KERNEL);
+	if (rc) {
+		cxl_destroy_snoop_filter(sf);
+
+		/*
+		 * The snoop filter insert failed due to a race, get the already
+		 * made snoop filter.
+		 */
+		if (rc == -EBUSY)
+			return cxl_snoop_filter_add_dport(dport, gid);
+
+		return rc;
+	}
+
+	return 0;
+}
+
+static void cxl_snoop_filter_free_capacity(void *data)
+{
+	struct cxl_cache_state *cstate = data;
+	struct cxl_snoop_filter *sf;
+
+	sf = xa_load(&cxl_snoop_filters, cstate->snoop_id);
+	if (!sf) {
+		pr_debug("failed to find snoop filter with id %d\n",
+			 cstate->snoop_id);
+		return;
+	}
+
+	guard(mutex)(&sf->lock);
+	sf->size += cstate->size;
+}
+
+/**
+ * devm_cxl_snoop_filter_alloc_capacity - Reserve snoop filter capacity for a
+ * CXL.cache device's cache.
+ * @cxlcd: Device to allocate capacity for
+ */
+int devm_cxl_snoop_filter_alloc_capacity(struct cxl_cachedev *cxlcd)
+{
+	struct cxl_dev_state *cxlds = cxlcd->cxlds;
+	struct cxl_cache_state *cstate = &cxlds->cstate;
+	struct cxl_snoop_filter *sf;
+	int gid = cstate->snoop_id;
+
+	sf = xa_load(&cxl_snoop_filters, gid);
+	if (!sf)
+		return -ENODEV;
+
+	guard(mutex)(&sf->lock);
+	if (cstate->size > sf->size)
+		return -EBUSY;
+
+	sf->size -= cstate->size;
+	return devm_add_action_or_reset(&cxlcd->dev,
+					cxl_snoop_filter_free_capacity, cstate);
+}
+EXPORT_SYMBOL_NS_GPL(devm_cxl_snoop_filter_alloc_capacity, "CXL");
+
+static void cxl_destroy_snoop_filters(void)
+{
+	struct cxl_snoop_filter *sf;
+	unsigned long gid;
+
+	xa_for_each(&cxl_snoop_filters, gid, sf)
+		cxl_destroy_snoop_filter(sf);
+
+	xa_destroy(&cxl_snoop_filters);
+}
+
+static int cxl_dport_setup_snoop_filter(struct cxl_dport *dport)
+{
+	struct cxl_port *port = dport->port;
+	struct cxl_port *parent_port;
+	u32 snoop, size;
+	int gid, rc;
+
+	/* Capability is only support by CXL root ports */
+	parent_port = parent_port_of(port);
+	if (!parent_port || !is_cxl_root(parent_port))
+		return 0;
+
+	if (!dport->reg_map.component_map.snoop.valid) {
+		dev_dbg(dport->dport_dev, "missing snoop filter capability\n");
+		return -ENXIO;
+	}
+
+	rc = cxl_map_component_regs(&dport->reg_map, &dport->regs.component,
+				    BIT(CXL_CM_CAP_CAP_ID_SNOOP));
+	if (rc)
+		return rc;
+
+	snoop = readl(dport->regs.snoop + CXL_SNOOP_GROUP_ID_OFFSET);
+	gid = FIELD_GET(CXL_SNOOP_GROUP_ID_MASK, snoop);
+
+	size = readl(dport->regs.snoop + CXL_SNOOP_FILTER_SIZE_OFFSET);
+	if (!size)
+		dev_warn(dport->dport_dev, "CXL snoop filter has no capacity\n");
+
+	rc = cxl_snoop_filter_add_dport(dport, gid);
+	if (rc) {
+		rc = cxl_create_snoop_filter(dport, gid, size);
+		if (rc)
+			return rc;
+	}
+
+	dport->snoop_id = gid;
+	return 0;
+}
+
 DEFINE_SHOW_ATTRIBUTE(einj_cxl_available_error_type);
 
 static int cxl_einj_inject(void *data, u64 type)
@@ -1177,6 +1348,7 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
 	dport->dport_dev = dport_dev;
 	dport->port_id = port_id;
 	dport->port = port;
+	dport->snoop_id = CXL_SNOOP_ID_NO_ID;
 
 	if (rcrb == CXL_RESOURCE_NONE) {
 		rc = cxl_dport_setup_regs(&port->dev, dport,
@@ -1207,6 +1379,11 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
 		dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
 			&component_reg_phys);
 
+	rc = cxl_dport_setup_snoop_filter(dport);
+	if (rc)
+		dev_dbg(dport->dport_dev,
+			"failed to set up snoop filter capability %d\n", rc);
+
 	cond_cxl_root_lock(port);
 	rc = add_dport(port, dport);
 	cond_cxl_root_unlock(port);
@@ -2585,6 +2762,7 @@ static void cxl_core_exit(void)
 	bus_unregister(&cxl_bus_type);
 	destroy_workqueue(cxl_bus_wq);
 	cxl_memdev_exit();
+	cxl_destroy_snoop_filters();
 	debugfs_remove_recursive(cxl_debugfs);
 }
 
diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c
index 5ca7b0eed568..280978b34d2f 100644
--- a/drivers/cxl/core/regs.c
+++ b/drivers/cxl/core/regs.c
@@ -92,6 +92,12 @@ void cxl_probe_component_regs(struct device *dev, void __iomem *base,
 			length = CXL_RAS_CAPABILITY_LENGTH;
 			rmap = &map->ras;
 			break;
+		case CXL_CM_CAP_CAP_ID_SNOOP:
+			dev_dbg(dev, "found Snoop Filter capability (0x%x)\n",
+				offset);
+			length = CXL_SNOOP_CAPABILITY_LENGTH;
+			rmap = &map->snoop;
+			break;
 		default:
 			dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id,
 				offset);
@@ -211,6 +217,7 @@ int cxl_map_component_regs(const struct cxl_register_map *map,
 	} mapinfo[] = {
 		{ &map->component_map.hdm_decoder, &regs->hdm_decoder },
 		{ &map->component_map.ras, &regs->ras },
+		{ &map->component_map.snoop, &regs->snoop },
 	};
 	int i;
 
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index e8d66de469d6..de29ffc3d74f 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -41,6 +41,7 @@ extern const struct nvdimm_security_ops *cxl_security_ops;
 
 #define   CXL_CM_CAP_CAP_ID_RAS 0x2
 #define   CXL_CM_CAP_CAP_ID_HDM 0x5
+#define   CXL_CM_CAP_CAP_ID_SNOOP 0x8
 #define   CXL_CM_CAP_CAP_HDM_VERSION 1
 
 /* HDM decoders CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure */
@@ -152,6 +153,12 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
 #define CXL_HEADERLOG_SIZE SZ_512
 #define CXL_HEADERLOG_SIZE_U32 SZ_512 / sizeof(u32)
 
+/* CXL 3.2 8.2.4.23 CXL Snoop Filter Capability Structure */
+#define CXL_SNOOP_GROUP_ID_OFFSET 0x0
+#define   CXL_SNOOP_GROUP_ID_MASK GENMASK(15, 0)
+#define CXL_SNOOP_FILTER_SIZE_OFFSET 0x4
+#define CXL_SNOOP_CAPABILITY_LENGTH 0x8
+
 /* CXL 2.0 8.2.8.1 Device Capabilities Array Register */
 #define CXLDEV_CAP_ARRAY_OFFSET 0x0
 #define   CXLDEV_CAP_ARRAY_CAP_ID 0
@@ -215,6 +222,7 @@ struct cxl_regs {
 	struct_group_tagged(cxl_component_regs, component,
 		void __iomem *hdm_decoder;
 		void __iomem *ras;
+		void __iomem *snoop;
 	);
 	/*
 	 * Common set of CXL Device register block base pointers
@@ -257,6 +265,7 @@ struct cxl_reg_map {
 struct cxl_component_reg_map {
 	struct cxl_reg_map hdm_decoder;
 	struct cxl_reg_map ras;
+	struct cxl_reg_map snoop;
 };
 
 struct cxl_device_reg_map {
@@ -693,6 +702,7 @@ struct cxl_dport {
 	struct access_coordinate coord[ACCESS_COORDINATE_MAX];
 	long link_latency;
 	int gpf_dvsec;
+	int snoop_id;
 };
 
 /**
@@ -759,6 +769,7 @@ struct cxl_dpa_info {
 	int nr_partitions;
 };
 
+#define CXL_SNOOP_ID_NO_ID (-1)
 
 /**
  * struct cxl_cache_state - State of a device's CXL cache
@@ -768,6 +779,7 @@ struct cxl_dpa_info {
 struct cxl_cache_state {
 	u64 size;
 	u32 unit;
+	int snoop_id;
 };
 
 /**
@@ -832,6 +844,8 @@ static inline struct cxl_dev_state *mbox_to_cxlds(struct cxl_mailbox *cxl_mbox)
 	return dev_get_drvdata(cxl_mbox->host);
 }
 
+int devm_cxl_snoop_filter_alloc_capacity(struct cxl_cachedev *cxlcd);
+
 /**
  * struct cxl_region_ref - track a region's interest in a port
  * @port: point in topology to install this reference
-- 
2.51.1