[RFC PATCH 6/6] pcie/cxl_timeout: Add CXL.mem Timeout & Isolation interrupt support

[Ben Cheatham <Benjamin.Cheatham@amd.com>]

Add support for CXL.mem Timeout & Isolation interrupts. A CXL root port
under isolation will not complete writes and will return an exception
response (i.e. poison) on reads (CXL 3.0 12.3.2). Therefore, when a
CXL-enabled PCIe root port enters isolation, we assume that the memory
under the port is unreachable and will need to be unmapped.

When an isolation interrupt occurs, the CXL Timeout & Isolation service
driver will mark the port (cxl_dport) as isolated, destroy any CXL memory
regions under the *bridge* (cxl_port), and attempt to unmap the memory
backing the CXL region(s). If the memory was already in use, the mapping
is not guaranteed to succeed.

Signed-off-by: Ben Cheatham <Benjamin.Cheatham@amd.com>
---
 drivers/cxl/core/pci.c         |   5 +
 drivers/cxl/core/port.c        |  80 ++++++++++++++++
 drivers/cxl/core/region.c      |   9 ++
 drivers/cxl/cxl.h              |  10 ++
 drivers/cxl/cxlpci.h           |   9 ++
 drivers/pci/pcie/cxl_timeout.c | 165 ++++++++++++++++++++++++++++++++-
 drivers/pci/pcie/portdrv.h     |   1 +
 7 files changed, 278 insertions(+), 1 deletion(-)

diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index 6c9c8d92f8f7..95b6a5f0d0cc 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -64,6 +64,11 @@ static int match_add_dports(struct pci_dev *pdev, void *data)
 	}
 	ctx->count++;
 
+	if (type == PCI_EXP_TYPE_ROOT_PORT && !pcie_cxlt_register_dport(dport))
+		return devm_add_action_or_reset(dport->dport_dev,
+						pcie_cxlt_unregister_dport,
+						dport);
+
 	return 0;
 }
 
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index e59d9d37aa65..88d114c67596 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -2000,6 +2000,86 @@ int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
 }
 EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL);
 
+/* Checks to see if a dport is above an endpoint */
+static bool cxl_dport_is_parent(struct cxl_dport *parent, struct cxl_ep *ep)
+{
+	struct cxl_dport *ep_dport = ep->dport;
+	struct cxl_port *ep_port = ep_dport->port;
+
+	while (!is_cxl_root(ep_port)) {
+		if (ep_dport == parent)
+			return true;
+
+		ep_dport = ep_port->parent_dport;
+		ep_port = ep_dport->port;
+	}
+
+	return false;
+}
+
+bool cxl_dport_is_in_region(struct cxl_dport *dport,
+			   struct cxl_region_ref *rr)
+{
+	struct cxl_region_params *p = &rr->region->params;
+	struct cxl_ep *ep;
+	int i;
+
+	for (i = 0; i < p->nr_targets; i++) {
+		if (!p->targets[i])
+			continue;
+
+		ep = cxl_ep_load(dport->port, cxled_to_memdev(p->targets[i]));
+
+		if (ep && cxl_dport_is_parent(dport, ep))
+			return true;
+	}
+
+	return false;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_dport_is_in_region, CXL);
+
+void cxl_port_kill_regions(struct cxl_port *port)
+{
+	struct cxl_endpoint_decoder *ep_decoder;
+	struct cxl_region_params *p;
+	struct cxl_region_ref *ref;
+	unsigned long index;
+	struct cxl_ep *ep;
+	int i;
+
+	xa_for_each(&port->regions, index, ref) {
+		p = &ref->region->params;
+
+		for (i = 0; i < p->nr_targets; i++) {
+			ep_decoder = p->targets[i];
+			if (!ep_decoder)
+				continue;
+
+			ep = cxl_ep_load(port, cxled_to_memdev(ep_decoder));
+			if (ep)
+				cxl_decoder_kill_region(ep_decoder);
+		}
+	}
+}
+EXPORT_SYMBOL_NS_GPL(cxl_port_kill_regions, CXL);
+
+bool cxl_port_is_isolated(struct cxl_port *port)
+{
+	struct cxl_dport *dport = port->parent_dport;
+
+	while (!is_cxl_root(port) && dport) {
+		if (dport->isolated || !dport->port)
+			return true;
+
+		dport = dport->port->parent_dport;
+		port = dport->port;
+	}
+
+
+	return false;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_port_is_isolated, CXL);
+
 /**
  * __cxl_driver_register - register a driver for the cxl bus
  * @cxl_drv: cxl driver structure to attach
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index 0f05692bfec3..f9aef17db26c 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -1699,6 +1699,12 @@ static int cxl_region_attach(struct cxl_region *cxlr,
 		return -ENXIO;
 	}
 
+	if (cxl_port_is_isolated(ep_port)) {
+		dev_err(&cxlr->dev, "%s:%s endpoint is under a dport in error isolation\n",
+			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
+		return -EBUSY;
+	}
+
 	if (cxled->cxld.target_type != cxlr->type) {
 		dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
 			dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
@@ -2782,6 +2788,9 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
 	struct resource *res;
 	int rc;
 
+	if (cxl_port_is_isolated(cxlmd->endpoint))
+		return ERR_PTR(-EBUSY);
+
 	do {
 		cxlr = __create_region(cxlrd, cxled->mode,
 				       atomic_read(&cxlrd->region_id));
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 3b5645ec95b9..1bee2560446a 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -138,6 +138,10 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
 #define   CXL_TIMEOUT_CONTROL_MEM_TIMEOUT_MASK GENMASK(3, 0)
 #define   CXL_TIMEOUT_CONTROL_MEM_TIMEOUT_ENABLE BIT(4)
 #define   CXL_TIMEOUT_CONTROL_MEM_ISO_ENABLE BIT(16)
+#define   CXL_TIMEOUT_CONTROL_MEM_INTR_ENABLE BIT(26)
+#define CXL_TIMEOUT_STATUS_OFFSET 0xC
+#define   CXL_TIMEOUT_STATUS_MEM_TIMEOUT BIT(0)
+#define   CXL_TIMEOUT_STATUS_MEM_ISO BIT(8)
 #define CXL_TIMEOUT_CAPABILITY_LENGTH 0x10
 
 /* CXL 3.0 8.2.4.23.2 CXL Timeout and Isolation Control Register, bits 3:0 */
@@ -700,8 +704,11 @@ struct cxl_dport {
 	struct access_coordinate sw_coord;
 	struct access_coordinate hb_coord;
 	long link_latency;
+	bool isolated;
 };
 
+bool cxl_port_is_isolated(struct cxl_port *port);
+
 /**
  * struct cxl_ep - track an endpoint's interest in a port
  * @ep: device that hosts a generic CXL endpoint (expander or accelerator)
@@ -735,6 +742,9 @@ struct cxl_region_ref {
 	int nr_targets;
 };
 
+bool cxl_dport_is_in_region(struct cxl_dport *dport, struct cxl_region_ref *ref);
+void cxl_port_kill_regions(struct cxl_port *port);
+
 /*
  * The platform firmware device hosting the root is also the top of the
  * CXL port topology. All other CXL ports have another CXL port as their
diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
index 711b05d9a370..7100e23a1819 100644
--- a/drivers/cxl/cxlpci.h
+++ b/drivers/cxl/cxlpci.h
@@ -106,4 +106,13 @@ void read_cdat_data(struct cxl_port *port);
 void cxl_cor_error_detected(struct pci_dev *pdev);
 pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
 				    pci_channel_state_t state);
+
+#ifdef CONFIG_PCIE_CXL_TIMEOUT
+int pcie_cxlt_register_dport(struct cxl_dport *dport);
+void pcie_cxlt_unregister_dport(struct cxl_dport *dport);
+#else
+int pcie_cxlt_register_dport(void *dport) { return -ENXIO; }
+void pcie_cxlt_unregister_dport(void *dport) {}
+#endif
+
 #endif /* __CXL_PCI_H__ */
diff --git a/drivers/pci/pcie/cxl_timeout.c b/drivers/pci/pcie/cxl_timeout.c
index 352d9370a999..2193e872b4b7 100644
--- a/drivers/pci/pcie/cxl_timeout.c
+++ b/drivers/pci/pcie/cxl_timeout.c
@@ -22,6 +22,8 @@
 
 #define NUM_CXL_TIMEOUT_RANGES 9
 
+static u32 num_cxlt_devs;
+
 struct cxl_timeout {
 	struct pcie_device *dev;
 	void __iomem *regs;
@@ -141,6 +143,141 @@ static struct pcie_cxlt_data *cxlt_create_pdata(struct pcie_device *dev)
 	return data;
 }
 
+int pcie_cxlt_register_dport(struct cxl_dport *dport)
+{
+	struct device *dev = dport->dport_dev;
+	struct pcie_device *pcie_dev;
+	struct pcie_cxlt_data *pdata;
+	struct pci_dev *pdev;
+
+	if (!dev_is_pci(dev))
+		return -ENXIO;
+
+	pdev = to_pci_dev(dev);
+
+	dev = pcie_port_find_device(pdev, PCIE_PORT_SERVICE_CXLT);
+	if (!dev) {
+		dev_warn(dev,
+			 "Device is not registered with cxl_timeout driver.\n");
+		return -ENODEV;
+	}
+
+	pcie_dev = to_pcie_device(dev);
+
+	pdata = get_service_data(pcie_dev);
+	pdata->dport = dport;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pcie_cxlt_register_dport);
+
+void pcie_cxlt_unregister_dport(struct cxl_dport *dport)
+{
+	struct device *dev = dport->dport_dev;
+	struct pcie_device *pcie_dev;
+	struct pcie_cxlt_data *pdata;
+	struct pci_dev *pdev;
+
+	if (!dev_is_pci(dev))
+		return;
+
+	pdev = to_pci_dev(dev);
+
+	dev = pcie_port_find_device(pdev, PCIE_PORT_SERVICE_CXLT);
+	if (!dev) {
+		dev_dbg(dev,
+			"Device was not registered with cxl_timeout driver.\n");
+		return;
+	}
+
+	pcie_dev = to_pcie_device(dev);
+	pdata = get_service_data(pcie_dev);
+	pdata->dport = NULL;
+}
+EXPORT_SYMBOL_GPL(pcie_cxlt_unregister_dport);
+
+struct cxl_timeout_wq_data {
+	struct work_struct w;
+	struct cxl_dport *dport;
+};
+
+static struct workqueue_struct *cxl_timeout_wq;
+
+static void cxl_timeout_handler(struct work_struct *w)
+{
+	struct cxl_timeout_wq_data *data =
+		container_of(w, struct cxl_timeout_wq_data, w);
+	struct cxl_dport *dport = data->dport;
+	struct cxl_port *port;
+	struct cxl_region_ref *ref;
+	unsigned long index;
+	bool kill_regions;
+
+	if (!dport || !dport->port)
+		return;
+
+	port = dport->port;
+
+	xa_for_each(&port->regions, index, ref)
+		if (cxl_dport_is_in_region(dport, ref))
+			kill_regions = true;
+
+	if (kill_regions)
+		cxl_port_kill_regions(port);
+
+	kfree(data);
+}
+
+irqreturn_t cxl_timeout_thread(int irq, void *data)
+{
+	struct cxl_timeout_wq_data *wq_data;
+	struct cxl_timeout *cxlt = data;
+	struct pcie_device *pcie_dev = cxlt->dev;
+	struct pcie_cxlt_data *pdata;
+	struct cxl_dport *dport;
+	u32 status;
+
+	/*
+	 * If the CXL core didn't register a cxl_dport with this PCIe device,
+	 * then dport enumeration failed and there's nothing to do CXL-wise.
+	 */
+	pdata = get_service_data(pcie_dev);
+	if (!pdata || !pdata->dport)
+		return IRQ_HANDLED;
+
+	dport = pdata->dport;
+
+	status = readl(cxlt->regs + CXL_TIMEOUT_STATUS_OFFSET);
+	if (!(status & CXL_TIMEOUT_STATUS_MEM_ISO
+	      || status & CXL_TIMEOUT_STATUS_MEM_TIMEOUT))
+		return IRQ_HANDLED;
+
+	dport->isolated = true;
+
+	wq_data = kzalloc(sizeof(struct cxl_timeout_wq_data), GFP_NOWAIT);
+	if (!wq_data)
+		return IRQ_NONE;
+
+	wq_data->dport = dport;
+
+	INIT_WORK(&wq_data->w, cxl_timeout_handler);
+	queue_work(cxl_timeout_wq, &wq_data->w);
+
+	return IRQ_HANDLED;
+}
+
+static int cxl_enable_interrupts(struct pcie_device *dev,
+				 struct cxl_timeout *cxlt)
+{
+	if (!cxlt || !FIELD_GET(CXL_TIMEOUT_CAP_INTR_SUPP, cxlt->cap))
+		return -ENXIO;
+
+	return devm_request_threaded_irq(&dev->device, dev->irq, NULL,
+					 cxl_timeout_thread,
+					 IRQF_SHARED | IRQF_ONESHOT, "cxltdrv",
+					 cxlt);
+}
+
 static bool cxl_validate_timeout_range(struct cxl_timeout *cxlt, u8 range)
 {
 	u8 timeout_ranges = FIELD_GET(CXL_TIMEOUT_CAP_MEM_TIMEOUT_MASK,
@@ -405,9 +542,28 @@ static int cxl_timeout_probe(struct pcie_device *dev)
 	if (rc && !timeout_enabled) {
 		pci_info(dev->port,
 			 "Failed to enable CXL.mem timeout and isolation.\n");
+		return rc;
 	}
 
-	return rc;
+	rc = cxl_enable_interrupts(dev, cxlt);
+	if (rc) {
+		pci_info(dev->port,
+			"Failed to enable CXL.mem timeout & isolation interrupts: %d\n",
+			rc);
+	} else {
+		pci_info(port, "enabled with IRQ %d\n", dev->irq);
+	}
+
+	num_cxlt_devs++;
+	return 0;
+}
+
+static void cxl_timeout_remove(struct pcie_device *dev)
+{
+	num_cxlt_devs--;
+
+	if (!num_cxlt_devs)
+		destroy_workqueue(cxl_timeout_wq);
 }
 
 static struct pcie_port_service_driver cxltdriver = {
@@ -416,6 +572,7 @@ static struct pcie_port_service_driver cxltdriver = {
 	.service	= PCIE_PORT_SERVICE_CXLT,
 
 	.probe		= cxl_timeout_probe,
+	.remove		= cxl_timeout_remove,
 	.driver		= {
 		.dev_groups = cxl_timeout_attribute_groups,
 	},
@@ -423,6 +580,12 @@ static struct pcie_port_service_driver cxltdriver = {
 
 int __init pcie_cxlt_init(void)
 {
+	cxl_timeout_wq = alloc_ordered_workqueue("cxl_timeout", 0);
+	if (!cxl_timeout_wq)
+		return -ENOMEM;
+
+	num_cxlt_devs = 0;
+
 	return pcie_port_service_register(&cxltdriver);
 }
 
diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h
index f89e7366e986..c56c629cf563 100644
--- a/drivers/pci/pcie/portdrv.h
+++ b/drivers/pci/pcie/portdrv.h
@@ -10,6 +10,7 @@
 #define _PORTDRV_H_
 
 #include <linux/compiler.h>
+#include <linux/errno.h>
 
 /* Service Type */
 #define PCIE_PORT_SERVICE_PME_SHIFT	0	/* Power Management Event */
-- 
2.34.1