[PATCH 13/15] cxl/cache: Add snoop filter allocation

[Ben Cheatham <Benjamin.Cheatham@amd.com>]

Add snoop filter capacity allocation for CXL.cache devices. CXL.cache
devices are expected to allocate snoop filter capacity equal to the size
of the address range they will cache using CXL.cache before using the
protocol (i.e. before allocating DMA space).

Devices may also indicate whether a CXL snoop filter being full is a
failure condition using the @strict_snoop member of struct
cxl_cache_state in struct cxl_dev_state. If any allocations on a filter
are made with @strict_snoop enabled, any allocations afterwards fail if
the filter is full regardless of @strict_snoop's setting.

A full snoop filter may cause performance degradation or unpredictable
behavior depending on the host's snoop filter implementation.

Signed-off-by: Ben Cheatham <Benjamin.Cheatham@amd.com>
---
 drivers/cxl/cache.c       |  26 +++++++-
 drivers/cxl/core/cache.c  | 125 +++++++++++++++++++++++++++++++++++++-
 drivers/cxl/core/memdev.c |   1 +
 include/cxl/cxl.h         |  13 ++++
 4 files changed, 162 insertions(+), 3 deletions(-)

diff --git a/drivers/cxl/cache.c b/drivers/cxl/cache.c
index 40d1e8330df7..b4574a76ac0b 100644
--- a/drivers/cxl/cache.c
+++ b/drivers/cxl/cache.c
@@ -179,6 +179,20 @@ struct cxl_cachedev *devm_cxl_add_cachedev(struct cxl_dev_state *cxlds)
 }
 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_cachedev, "CXL");
 
+static int cxl_cachedev_find_snoop_gid(struct cxl_cachedev *cxlcd)
+{
+	struct cxl_dport *iter;
+
+	for (iter = cxlcd->endpoint->parent_dport;
+	     iter && !is_cxl_root(iter->port);
+	     iter = iter->port->parent_dport) {
+		if (iter->snoop != CXL_SNOOP_FILTER_NO_GROUP_ID)
+			return iter->snoop;
+	}
+
+	return -ENXIO;
+}
+
 static int cxl_cache_probe(struct device *dev)
 {
 	struct cxl_cachedev *cxlcd = to_cxl_cachedev(dev);
@@ -236,7 +250,17 @@ static int cxl_cache_probe(struct device *dev)
 	if (rc)
 		return rc;
 
-	return devm_add_action_or_reset(dev, deprogram_cache_id, cxlcd);
+	rc = devm_add_action_or_reset(dev, deprogram_cache_id, cxlcd);
+ 	if (rc)
+ 		return rc;
+
+	rc = cxl_cachedev_find_snoop_gid(cxlcd);
+	if (rc < 0)
+		return rc;
+
+	cxlcd->cxlds->cstate.gid = rc;
+
+ 	return 0;
 }
 
 static struct cxl_driver cxl_cache_driver = {
diff --git a/drivers/cxl/core/cache.c b/drivers/cxl/core/cache.c
index f07c49cfacec..429c997b2ada 100644
--- a/drivers/cxl/core/cache.c
+++ b/drivers/cxl/core/cache.c
@@ -571,17 +571,28 @@ void cxl_cachedev_deprogram_cache_id(struct cxl_cachedev *cxlcd)
 }
 EXPORT_SYMBOL_FOR_MODULES(cxl_cachedev_deprogram_cache_id, "cxl_cache");
 
+struct snoop_allocation {
+	struct cxl_cachedev *cxlcd;
+	bool strict;
+	u64 size;
+	int id;
+};
+
 /**
  * struct cxl_snoop_filter - CXL snoop filter instance for tracking CXL.cache
  * devices below a dport
  *
+ * @allocations: Allocations on this filter
  * @lock: Used for allocations
+ * @strict: Number of allocations from devices that disallow oversubscription
  * @avail: Available capacity left in the filter
  * @size: Size of the filter
  * @id: Group ID of the filter
  */
 struct cxl_snoop_filter {
+	struct xarray allocations;
 	struct mutex lock;
+	u32 strict;
 	u64 avail;
 	u64 size;
 	int id;
@@ -597,15 +608,37 @@ static struct cxl_snoop_filter *create_snoop_filter(u64 size, int id)
 
 	sf->id = id;
 	sf->size = sf->avail = size;
+	xa_init(&sf->allocations);
 	mutex_init(&sf->lock);
 
 	return sf;
 }
 
-static void destroy_snoop_filter(struct cxl_snoop_filter *sf)
+static void free_snoop_capacity(void *);
+
+static void __free_snoop_capacity(struct snoop_allocation *alloc)
 {
-	lockdep_assert_held_write(&snoop_rwsem);
+	struct cxl_snoop_filter *sf;
+
+	sf = xa_load(&snoop_filters, alloc->id);
+	if (sf) {
+		scoped_guard(mutex, &sf->lock) {
+			sf->avail += alloc->size;
+
+			if (alloc->strict)
+				sf->strict--;
+
+			xa_erase(&sf->allocations, (unsigned long)alloc);
+		}
+	}
 
+	put_device(&alloc->cxlcd->dev);
+	kfree(alloc);
+}
+
+static void destroy_snoop_filter(struct cxl_snoop_filter *sf)
+{
+	xa_destroy(&sf->allocations);
 	mutex_destroy(&sf->lock);
 	kfree(sf);
 }
@@ -638,6 +671,94 @@ static struct cxl_snoop_filter *find_or_add_snoop_filter(u64 size, int id)
 	return sf;
 }
 
+static struct snoop_allocation *
+snoop_alloc_capacity(struct cxl_snoop_filter *sf, struct cxl_cachedev *cxlcd,
+		     u64 size)
+{
+	struct cxl_cache_state *cstate = &cxlcd->cxlds->cstate;
+	struct snoop_allocation *alloc;
+	int rc;
+
+	guard(mutex)(&sf->lock);
+	if (sf->avail < size) {
+		/*
+		 * Either this device or a device already using the filter
+		 * can't use an oversubscribed filter
+		 */
+		if (cstate->strict_snoop || sf->strict > 0)
+			return ERR_PTR(-ENOSPC);
+
+		size = sf->avail;
+	}
+
+	alloc = kmalloc_obj(*alloc);
+	if (!alloc)
+		return ERR_PTR(-ENOMEM);
+
+	*alloc = (struct snoop_allocation) {
+		.cxlcd = cxlcd,
+		.size = size,
+		.strict = cstate->strict_snoop,
+		.id = sf->id,
+	};
+
+	rc = xa_insert(&sf->allocations, (unsigned long)alloc, alloc,
+		       GFP_KERNEL);
+	if (rc) {
+		kfree(alloc);
+		return ERR_PTR(rc);
+	}
+
+	if (cstate->strict_snoop)
+		sf->strict++;
+
+	sf->avail -= size;
+	return alloc;
+}
+
+static void free_snoop_capacity(void *alloc)
+{
+	guard(rwsem_read)(&snoop_rwsem);
+	__free_snoop_capacity(alloc);
+}
+
+/**
+ * devm_cxl_cachedev_alloc_snoop_capacity - Allocate space in the system's snoop
+ * filter for a given CXL.cache device
+ * @cxlcd: Cache device to allocate capacity for
+ * @size: Size of allocation
+ *
+ * NOTE: CXL accelerator drivers are expected to call this function before
+ * using CXL.cache to access host memory. Failure to do so may result in
+ * unpredictable or undesirable behavior depending on the host's snoop filter
+ * implementation.
+ */
+int devm_cxl_cachedev_alloc_snoop_capacity(struct cxl_cachedev *cxlcd, u64 size)
+{
+	struct cxl_cache_state *cstate = &cxlcd->cxlds->cstate;
+	struct snoop_allocation *alloc;
+	struct cxl_snoop_filter *sf;
+
+	if (cstate->gid < 0)
+		return -EINVAL;
+
+	scoped_guard(rwsem_read, &snoop_rwsem) {
+		sf = xa_load(&snoop_filters, cstate->gid);
+		if (!sf)
+			return -ENODEV;
+
+		alloc = snoop_alloc_capacity(sf, cxlcd, size);
+		if (IS_ERR_OR_NULL(alloc))
+			return !alloc ? -ENOMEM : PTR_ERR(alloc);
+
+		get_device(&cxlcd->dev);
+	}
+
+	return devm_add_action_or_reset(&cxlcd->dev, free_snoop_capacity,
+					alloc);
+}
+EXPORT_SYMBOL_NS_GPL(devm_cxl_cachedev_alloc_snoop_capacity, "CXL");
+
 int cxl_dport_probe_snoop_filter(struct cxl_dport *dport)
 {
 	struct cxl_snoop_filter *sf;
diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c
index b3419df586b9..e169301bbfcf 100644
--- a/drivers/cxl/core/memdev.c
+++ b/drivers/cxl/core/memdev.c
@@ -751,6 +751,7 @@ struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev,
 	cxlds->cxl_dvsec = dvsec;
 	cxlds->reg_map.host = dev;
 	cxlds->reg_map.resource = CXL_RESOURCE_NONE;
+	cxlds->cstate.gid = CXL_SNOOP_FILTER_NO_GROUP_ID;
 
 	if (has_mbox)
 		cxlds->cxl_mbox.host = dev;
diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h
index 22b9c8c9c06e..5d550dd70870 100644
--- a/include/cxl/cxl.h
+++ b/include/cxl/cxl.h
@@ -165,6 +165,8 @@ struct cxl_dpa_partition {
  * struct cxl_cache_state - CXL cache device state for use by external drivers
  * @size: Size of device's cache
  * @unit: Unit of device's cache in bytes
+ * @strict_snoop: Whether a full snoop filter should fail allocations
+ * @gid: Group ID of the snoop filter this device belongs to
  */
 struct cxl_cache_state {
 	/* Public for endpoint drivers */
@@ -175,6 +177,12 @@ struct cxl_cache_state {
 	 */
 	u64 size;
 	u64 unit;
+
+	/* Should be set by endpoint drivers */
+	bool strict_snoop;
+
+	/* Private for endpoint drivers */
+	int gid;
 };
 
 /**
@@ -264,9 +272,14 @@ int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity);
 
 #if IS_ENABLED(CONFIG_CXL_CACHE)
 struct cxl_cachedev *devm_cxl_add_cachedev(struct cxl_dev_state *cxlds);
+int devm_cxl_cachedev_alloc_snoop_capacity(struct cxl_cachedev *cxlcd,
+					   u64 size);
 #else
 static inline struct cxl_cachedev *
 devm_cxl_add_cachedev(struct cxl_dev_state *cxlds)
 { return ERR_PTR(-ENXIO); }
+static inline int
+devm_cxl_cachedev_alloc_snoop_capacity(struct cxl_cachedev *cxlcd, u64 size)
+{ return -ENXIO; }
 #endif /* CONFIG_CXL_CACHE */
 #endif /* __CXL_CXL_H__ */
-- 
2.53.0