[RFC PATCH v2 4/5] PCI/TSM: Support multiple coherent ranges via coh_idx

[Ankit Agrawal <ankita@nvidia.com>]

Devices with multiple decoders can report more than one coherent
window in a single interface report. Generalize the single
0xffff sentinel into a small id space:
PCI_TSM_DEVIF_REPORT_MMIO_RANGE_ID_TO_COH_IDX() treats any range id in
the top PCI_CXL_MAX_COHERENT_RANGES values (counting down from ~0) as
a coherent range index coh_idx. This keeps id 0xffff (~0) backward
compatible with the single-range case (coh_idx 0) while allowing ~1,
~2, and so on for additional windows.

Only id 0xffff (~0) is spec-defined: Arm RME System Architecture
(DEN0129) section B2.3.5.2 reserves range ID 0xFFFF for a device's
region range. ~1, ~2, and so on are this tree's own extension of that
idea, invented to cover multi-range reporting.

Use coh_idx through pci_tsm_retrieve_coherent_range() to select the coh_idx
entry of pdev->coh_resource[], the precommitted-decoder snapshot and
relax pci_tsm_mmio_alloc() to accept more than one coherent region.
The coherent-range entries follow all BAR entries in the report
and allow multiple trailing coherent ranges.

Signed-off-by: Ankit Agrawal <ankita@nvidia.com>
---
 drivers/pci/tsm.c       | 36 ++++++++++++++++++++++++------------
 include/linux/pci-tsm.h | 34 +++++++++++++++++++---------------
 2 files changed, 43 insertions(+), 27 deletions(-)

diff --git a/drivers/pci/tsm.c b/drivers/pci/tsm.c
index bc3218c58ff0..a2a25bafc93e 100644
--- a/drivers/pci/tsm.c
+++ b/drivers/pci/tsm.c
@@ -1004,7 +1004,14 @@ EXPORT_SYMBOL_GPL(pci_tsm_mmio_teardown);
 #define PCI_TSM_DEVIF_REPORT_MMIO_ATTR_IS_NON_TEE BIT(2)
 #define PCI_TSM_DEVIF_REPORT_MMIO_ATTR_IS_UPDATABLE BIT(3)
 #define PCI_TSM_DEVIF_REPORT_MMIO_ATTR_RANGE_ID GENMASK(31, 16)
-#define PCI_TSM_DEVIF_REPORT_MMIO_RANGE_ID_COHERENT 0xffff
+
+/*
+ * Only id ~0 (0xffff) is spec-defined (Arm RME DEN0129 section B2.3.5.2);
+ * ~1, ~2, etc. extend it for multi-range reporting: id ~N selects coherent
+ * window N, keeping coherent ids disjoint from BAR ids
+ * (0..PCI_STD_NUM_BARS-1) without a separate flag.
+ */
+#define PCI_TSM_DEVIF_REPORT_MMIO_RANGE_ID_TO_COH_IDX(id) ((u16)~(id))
 
 /* An interface report 'pfn' is 4K in size */
 struct pci_tsm_devif_mmio {
@@ -1042,6 +1049,7 @@ struct pci_tsm_mmio *pci_tsm_mmio_alloc(struct pci_dev *pdev,
 	u64 reporting_bar_base, last_reporting_end;
 	u32 mmio_range_count;
 	int last_bar = -1;
+	bool seen_coherent = false;
 	int i;
 
 	if (report_len < sizeof(*devif_report))
@@ -1071,28 +1079,32 @@ struct pci_tsm_mmio *pci_tsm_mmio_alloc(struct pci_dev *pdev,
 		u32 attr = __le32_to_cpu(mmio_data->attributes);
 		int bar = FIELD_GET(PCI_TSM_DEVIF_REPORT_MMIO_ATTR_RANGE_ID,
 				    attr);
-		bool coherent =
-			bar == PCI_TSM_DEVIF_REPORT_MMIO_RANGE_ID_COHERENT;
+		unsigned int coh_idx =
+			PCI_TSM_DEVIF_REPORT_MMIO_RANGE_ID_TO_COH_IDX(bar);
+		bool coherent = coh_idx < PCI_CXL_MAX_COHERENT_RANGES;
 
 		if (coherent) {
-			if (i != mmio_range_count - 1) {
-				pci_dbg(pdev, "Coherent reporting range is not last\n");
-				return NULL;
-			}
+			seen_coherent = true;
 
 			/*
-			 * No BAR names the coherent range, so fail closed if
-			 * neither a CXL region nor a committed HDM decoder
-			 * resolves its address.
+			 * Fail closed if pdev->coh_resource[coh_idx] isn't
+			 * populated: nothing else in the report identifies
+			 * the window's address.
 			 */
-			if (pci_tsm_retrieve_coherent_range(pdev, &window)) {
-				pci_dbg(pdev, "No CXL region or committed HDM decoder for coherent reporting range\n");
+			if (pci_tsm_retrieve_coherent_range(pdev, coh_idx, &window)) {
+				pci_dbg(pdev, "No populated coh_resource[] entry for coherent range %u\n",
+					coh_idx);
 				return NULL;
 			}
 
 			reporting_bar_base = tsm_offset &
 					     ~((u64)range_len(&window) - 1);
 		} else {
+			if (seen_coherent) {
+				pci_dbg(pdev, "BAR reporting range follows a coherent range\n");
+				return NULL;
+			}
+
 			if (bar >= PCI_STD_NUM_BARS ||
 			    !(pci_resource_flags(pdev, bar) & IORESOURCE_MEM) ||
 			    (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET)) {
diff --git a/include/linux/pci-tsm.h b/include/linux/pci-tsm.h
index 7c29f10e7795..ef4edf3efaf3 100644
--- a/include/linux/pci-tsm.h
+++ b/include/linux/pci-tsm.h
@@ -261,32 +261,35 @@ static inline bool pci_tsm_coh_resource_contains(struct pci_dev *pdev,
 }
 
 /*
- * pci_tsm_retrieve_coherent_range() - resolve the device's coherent CXL window
+ * pci_tsm_retrieve_coherent_range() - resolve the device coherent CXL window
  * @pdev: device owner of the reported ranges
+ * @coh_idx: which coherent range to resolve, per
+ *	     %PCI_TSM_DEVIF_REPORT_MMIO_RANGE_ID_TO_COH_IDX()
  * @out: host physical (guest IPA) range of the coherent window
  *
- * The coherent range carries no BAR number, so its address comes from
- * pdev->coh_resource[], a pre-driver-bind snapshot of committed CXL HDM
- * decoders.
+ * Each @coh_idx names one of the device coherent (CXL) windows. It carries
+ * no BAR number and so its address comes from pdev->coh_resource[@coh_idx],
+ * which is a snapshot of committed CXL HDM decoders regions.
  *
- * Return: 0 with *@out set from the first populated entry, or -ENODEV if no
- * entry is populated.
+ * Return: 0 with *@out set, or -ENODEV if @coh_idx is out
+ * of range or has no populated coh_resource[] entry.
  */
 static inline int pci_tsm_retrieve_coherent_range(struct pci_dev *pdev,
+						  unsigned int coh_idx,
 						  struct range *out)
 {
-	for (int i = 0; i < PCI_CXL_MAX_COHERENT_RANGES; i++) {
-		struct resource *res = &pdev->coh_resource[i];
+	struct resource *res;
 
-		if (resource_type(res) != IORESOURCE_MEM)
-			continue;
+	if (coh_idx >= PCI_CXL_MAX_COHERENT_RANGES)
+		return -ENODEV;
 
-		out->start = res->start;
-		out->end = res->end;
-		return 0;
-	}
+	res = &pdev->coh_resource[coh_idx];
+	if (resource_type(res) != IORESOURCE_MEM)
+		return -ENODEV;
 
-	return -ENODEV;
+	out->start = res->start;
+	out->end = res->end;
+	return 0;
 }
 #else
 static inline bool pci_tsm_coh_resource_contains(struct pci_dev *pdev,
@@ -296,6 +299,7 @@ static inline bool pci_tsm_coh_resource_contains(struct pci_dev *pdev,
 }
 
 static inline int pci_tsm_retrieve_coherent_range(struct pci_dev *pdev,
+						  unsigned int coh_idx,
 						  struct range *out)
 {
 	return -ENODEV;
-- 
2.53.0