[PATCH 00/16] PCI: MSI parent domain conversion

[PATCH 00/16] PCI: MSI parent domain conversion

[Nam Cao]

The initial implementation of PCI/MSI interrupt domains in the hierarchical
interrupt domain model used a shortcut by providing a global PCI/MSI
domain.

This works because the PCI/MSI[X] hardware is standardized and uniform, but
it violates the basic design principle of hierarchical interrupt domains:
Each hardware block involved in the interrupt delivery chain should have a
separate interrupt domain.

For PCI/MSI[X], the interrupt controller is per PCI device and not a global
made-up entity.

Unsurprisingly, the shortcut turned out to have downsides as it does not
allow dynamic allocation of interrupt vectors after initialization and it
prevents supporting IMS on PCI. For further details, see:

https://lore.kernel.org/lkml/20221111120501.026511281@linutronix.de/

The solution is implementing per device MSI domains, this means the
entities which provide global PCI/MSI domain so far have to implement MSI
parent domain functionality instead.

This series converts the PCI controller drivers to implement MSI parent
domain.

 drivers/pci/Kconfig                           |   1 +
 drivers/pci/controller/Kconfig                |  11 +
 drivers/pci/controller/dwc/Kconfig            |   1 +
 .../pci/controller/dwc/pcie-designware-host.c |  68 ++----
 drivers/pci/controller/dwc/pcie-designware.h  |   1 -
 drivers/pci/controller/mobiveil/Kconfig       |   1 +
 .../controller/mobiveil/pcie-mobiveil-host.c  |  42 ++--
 .../pci/controller/mobiveil/pcie-mobiveil.h   |   1 -
 drivers/pci/controller/pci-aardvark.c         |  59 ++---
 drivers/pci/controller/pci-hyperv.c           |  98 ++++++--
 drivers/pci/controller/pcie-altera-msi.c      |  44 ++--
 drivers/pci/controller/pcie-brcmstb.c         |  44 ++--
 drivers/pci/controller/pcie-iproc-msi.c       |  45 ++--
 drivers/pci/controller/pcie-mediatek-gen3.c   |  67 ++---
 drivers/pci/controller/pcie-mediatek.c        |  46 ++--
 drivers/pci/controller/pcie-rcar-host.c       |  69 ++----
 drivers/pci/controller/pcie-xilinx-dma-pl.c   |  48 ++--
 drivers/pci/controller/pcie-xilinx-nwl.c      |  45 ++--
 drivers/pci/controller/pcie-xilinx.c          |  55 +++--
 drivers/pci/controller/plda/Kconfig           |   1 +
 drivers/pci/controller/plda/pcie-plda-host.c  |  44 ++--
 drivers/pci/controller/plda/pcie-plda.h       |   1 -
 drivers/pci/controller/vmd.c                  | 229 +++++++++---------
 23 files changed, 504 insertions(+), 517 deletions(-)

-- 
2.39.5

[PATCH 01/16] PCI: dwc: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Jingoo Han <jingoohan1@gmail.com>
---
 drivers/pci/controller/dwc/Kconfig            |  1 +
 .../pci/controller/dwc/pcie-designware-host.c | 68 +++++++------------
 drivers/pci/controller/dwc/pcie-designware.h  |  1 -
 3 files changed, 25 insertions(+), 45 deletions(-)

diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
index bb95877b2c6c4..8a63e9bc0c039 100644
--- a/drivers/pci/controller/dwc/Kconfig
+++ b/drivers/pci/controller/dwc/Kconfig
@@ -19,6 +19,7 @@ config PCIE_DW_DEBUGFS
 config PCIE_DW_HOST
 	bool
 	select PCIE_DW
+	select IRQ_MSI_LIB
 
 config PCIE_DW_EP
 	bool
diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index 906277f9ffaf7..a953e07a68aff 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -10,6 +10,7 @@
 
 #include <linux/iopoll.h>
 #include <linux/irqchip/chained_irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/msi.h>
 #include <linux/of_address.h>
@@ -23,35 +24,21 @@
 static struct pci_ops dw_pcie_ops;
 static struct pci_ops dw_child_pcie_ops;
 
-static void dw_msi_ack_irq(struct irq_data *d)
-{
-	irq_chip_ack_parent(d);
-}
-
-static void dw_msi_mask_irq(struct irq_data *d)
-{
-	pci_msi_mask_irq(d);
-	irq_chip_mask_parent(d);
-}
-
-static void dw_msi_unmask_irq(struct irq_data *d)
-{
-	pci_msi_unmask_irq(d);
-	irq_chip_unmask_parent(d);
-}
-
-static struct irq_chip dw_pcie_msi_irq_chip = {
-	.name = "PCI-MSI",
-	.irq_ack = dw_msi_ack_irq,
-	.irq_mask = dw_msi_mask_irq,
-	.irq_unmask = dw_msi_unmask_irq,
-};
-
-static struct msi_domain_info dw_pcie_msi_domain_info = {
-	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		  MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX |
-		  MSI_FLAG_MULTI_PCI_MSI,
-	.chip	= &dw_pcie_msi_irq_chip,
+#define DW_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS		| \
+				    MSI_FLAG_USE_DEF_CHIP_OPS		| \
+				    MSI_FLAG_NO_AFFINITY		| \
+				    MSI_FLAG_PCI_MSI_MASK_PARENT)
+#define DW_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI		| \
+				     MSI_FLAG_PCI_MSIX			| \
+				     MSI_GENERIC_FLAGS_MASK)
+
+static const struct msi_parent_ops dw_pcie_msi_parent_ops = {
+	.required_flags		= DW_PCIE_MSI_FLAGS_REQUIRED,
+	.supported_flags	= DW_PCIE_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
+	.prefix			= "DW-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 };
 
 /* MSI int handler */
@@ -228,25 +215,19 @@ int dw_pcie_allocate_domains(struct dw_pcie_rp *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct fwnode_handle *fwnode = of_fwnode_handle(pci->dev->of_node);
-
-	pp->irq_domain = irq_domain_create_linear(fwnode, pp->num_vectors,
-					       &dw_pcie_msi_domain_ops, pp);
+	struct irq_domain_info info = {
+		.fwnode		= fwnode,
+		.ops		= &dw_pcie_msi_domain_ops,
+		.size		= pp->num_vectors,
+		.host_data	= pp,
+	};
+
+	pp->irq_domain = msi_create_parent_irq_domain(&info, &dw_pcie_msi_parent_ops);
 	if (!pp->irq_domain) {
 		dev_err(pci->dev, "Failed to create IRQ domain\n");
 		return -ENOMEM;
 	}
 
-	irq_domain_update_bus_token(pp->irq_domain, DOMAIN_BUS_NEXUS);
-
-	pp->msi_domain = pci_msi_create_irq_domain(fwnode,
-						   &dw_pcie_msi_domain_info,
-						   pp->irq_domain);
-	if (!pp->msi_domain) {
-		dev_err(pci->dev, "Failed to create MSI domain\n");
-		irq_domain_remove(pp->irq_domain);
-		return -ENOMEM;
-	}
-
 	return 0;
 }
 
@@ -260,7 +241,6 @@ static void dw_pcie_free_msi(struct dw_pcie_rp *pp)
 							 NULL, NULL);
 	}
 
-	irq_domain_remove(pp->msi_domain);
 	irq_domain_remove(pp->irq_domain);
 }
 
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index ce9e18554e426..d9daee4ce220d 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -417,7 +417,6 @@ struct dw_pcie_rp {
 	const struct dw_pcie_host_ops *ops;
 	int			msi_irq[MAX_MSI_CTRLS];
 	struct irq_domain	*irq_domain;
-	struct irq_domain	*msi_domain;
 	dma_addr_t		msi_data;
 	struct irq_chip		*msi_irq_chip;
 	u32			num_vectors;
-- 
2.39.5

[PATCH 02/16] PCI: mobiveil: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Karthikeyan Mitran <m.karthikeyan@mobiveil.co.in>
Cc: Hou Zhiqiang <Zhiqiang.Hou@nxp.com>
---
 drivers/pci/controller/mobiveil/Kconfig       |  1 +
 .../controller/mobiveil/pcie-mobiveil-host.c  | 42 ++++++++++---------
 .../pci/controller/mobiveil/pcie-mobiveil.h   |  1 -
 3 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/drivers/pci/controller/mobiveil/Kconfig b/drivers/pci/controller/mobiveil/Kconfig
index 58ce034f701ab..c50c4625937f8 100644
--- a/drivers/pci/controller/mobiveil/Kconfig
+++ b/drivers/pci/controller/mobiveil/Kconfig
@@ -9,6 +9,7 @@ config PCIE_MOBIVEIL
 config PCIE_MOBIVEIL_HOST
 	bool
 	depends on PCI_MSI
+	select IRQ_MSI_LIB
 	select PCIE_MOBIVEIL
 
 config PCIE_LAYERSCAPE_GEN4
diff --git a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c
index cd44ddb698ea2..d17e887b6b61d 100644
--- a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c
+++ b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c
@@ -12,6 +12,7 @@
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
@@ -353,16 +354,19 @@ static const struct irq_domain_ops intx_domain_ops = {
 	.map = mobiveil_pcie_intx_map,
 };
 
-static struct irq_chip mobiveil_msi_irq_chip = {
-	.name = "Mobiveil PCIe MSI",
-	.irq_mask = pci_msi_mask_irq,
-	.irq_unmask = pci_msi_unmask_irq,
-};
+#define MOBIVEIL_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS		| \
+				     MSI_FLAG_USE_DEF_CHIP_OPS		| \
+				     MSI_FLAG_NO_AFFINITY)
+
+#define MOBIVEIL_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK		| \
+				      MSI_FLAG_PCI_MSIX)
 
-static struct msi_domain_info mobiveil_msi_domain_info = {
-	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		  MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX,
-	.chip	= &mobiveil_msi_irq_chip,
+static const struct msi_parent_ops mobiveil_msi_parent_ops = {
+	.required_flags		= MOBIVEIL_MSI_FLAGS_REQUIRED,
+	.supported_flags	= MOBIVEIL_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.prefix			= "Mobiveil-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 };
 
 static void mobiveil_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
@@ -439,19 +443,17 @@ static int mobiveil_allocate_msi_domains(struct mobiveil_pcie *pcie)
 	struct mobiveil_msi *msi = &pcie->rp.msi;
 
 	mutex_init(&msi->lock);
-	msi->dev_domain = irq_domain_create_linear(NULL, msi->num_of_vectors,
-						   &msi_domain_ops, pcie);
-	if (!msi->dev_domain) {
-		dev_err(dev, "failed to create IRQ domain\n");
-		return -ENOMEM;
-	}
 
-	msi->msi_domain = pci_msi_create_irq_domain(fwnode,
-						    &mobiveil_msi_domain_info,
-						    msi->dev_domain);
-	if (!msi->msi_domain) {
+	struct irq_domain_info info = {
+		.fwnode		= fwnode,
+		.ops		= &msi_domain_ops,
+		.host_data	= pcie,
+		.size		= msi->num_of_vectors,
+	};
+
+	msi->dev_domain = msi_create_parent_irq_domain(&info, &mobiveil_msi_parent_ops);
+	if (!msi->dev_domain) {
 		dev_err(dev, "failed to create MSI domain\n");
-		irq_domain_remove(msi->dev_domain);
 		return -ENOMEM;
 	}
 
diff --git a/drivers/pci/controller/mobiveil/pcie-mobiveil.h b/drivers/pci/controller/mobiveil/pcie-mobiveil.h
index 662f17f9bf65c..7246de6a71768 100644
--- a/drivers/pci/controller/mobiveil/pcie-mobiveil.h
+++ b/drivers/pci/controller/mobiveil/pcie-mobiveil.h
@@ -135,7 +135,6 @@
 
 struct mobiveil_msi {			/* MSI information */
 	struct mutex lock;		/* protect bitmap variable */
-	struct irq_domain *msi_domain;
 	struct irq_domain *dev_domain;
 	phys_addr_t msi_pages_phys;
 	int num_of_vectors;
-- 
2.39.5

[PATCH 03/16] PCI: aardvark: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com>
Cc: "Pali Rohár" <pali@kernel.org>
Cc: linux-arm-kernel@lists.infradead.org
---
 drivers/pci/controller/Kconfig        |  1 +
 drivers/pci/controller/pci-aardvark.c | 59 +++++++++++----------------
 2 files changed, 24 insertions(+), 36 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 886f6f43a895f..91a2d4ffc3ac4 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -13,6 +13,7 @@ config PCI_AARDVARK
 	depends on OF
 	depends on PCI_MSI
 	select PCI_BRIDGE_EMUL
+	select IRQ_MSI_LIB
 	help
 	 Add support for Aardvark 64bit PCIe Host Controller. This
 	 controller is part of the South Bridge of the Marvel Armada
diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c
index 7bac64533b143..e34bea1ff0ac6 100644
--- a/drivers/pci/controller/pci-aardvark.c
+++ b/drivers/pci/controller/pci-aardvark.c
@@ -13,6 +13,7 @@
 #include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -278,7 +279,6 @@ struct advk_pcie {
 	struct irq_domain *irq_domain;
 	struct irq_chip irq_chip;
 	raw_spinlock_t irq_lock;
-	struct irq_domain *msi_domain;
 	struct irq_domain *msi_inner_domain;
 	raw_spinlock_t msi_irq_lock;
 	DECLARE_BITMAP(msi_used, MSI_IRQ_NUM);
@@ -1332,18 +1332,6 @@ static void advk_msi_irq_unmask(struct irq_data *d)
 	raw_spin_unlock_irqrestore(&pcie->msi_irq_lock, flags);
 }
 
-static void advk_msi_top_irq_mask(struct irq_data *d)
-{
-	pci_msi_mask_irq(d);
-	irq_chip_mask_parent(d);
-}
-
-static void advk_msi_top_irq_unmask(struct irq_data *d)
-{
-	pci_msi_unmask_irq(d);
-	irq_chip_unmask_parent(d);
-}
-
 static struct irq_chip advk_msi_bottom_irq_chip = {
 	.name			= "MSI",
 	.irq_compose_msi_msg	= advk_msi_irq_compose_msi_msg,
@@ -1436,17 +1424,20 @@ static const struct irq_domain_ops advk_pcie_irq_domain_ops = {
 	.xlate = irq_domain_xlate_onecell,
 };
 
-static struct irq_chip advk_msi_irq_chip = {
-	.name		= "advk-MSI",
-	.irq_mask	= advk_msi_top_irq_mask,
-	.irq_unmask	= advk_msi_top_irq_unmask,
-};
-
-static struct msi_domain_info advk_msi_domain_info = {
-	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		  MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI |
-		  MSI_FLAG_PCI_MSIX,
-	.chip	= &advk_msi_irq_chip,
+#define ADVK_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
+				 MSI_FLAG_USE_DEF_CHIP_OPS	| \
+				 MSI_FLAG_PCI_MSI_MASK_PARENT	| \
+				 MSI_FLAG_NO_AFFINITY)
+#define ADVK_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK	| \
+				  MSI_FLAG_PCI_MSIX		| \
+				  MSI_FLAG_MULTI_PCI_MSI)
+
+static const struct msi_parent_ops advk_msi_parent_ops = {
+	.required_flags		= ADVK_MSI_FLAGS_REQUIRED,
+	.supported_flags	= ADVK_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.prefix			= "advk-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 };
 
 static int advk_pcie_init_msi_irq_domain(struct advk_pcie *pcie)
@@ -1456,26 +1447,22 @@ static int advk_pcie_init_msi_irq_domain(struct advk_pcie *pcie)
 	raw_spin_lock_init(&pcie->msi_irq_lock);
 	mutex_init(&pcie->msi_used_lock);
 
-	pcie->msi_inner_domain = irq_domain_create_linear(NULL, MSI_IRQ_NUM,
-							  &advk_msi_domain_ops, pcie);
-	if (!pcie->msi_inner_domain)
-		return -ENOMEM;
+	struct irq_domain_info info = {
+		.fwnode		= dev_fwnode(dev),
+		.ops		= &advk_msi_domain_ops,
+		.host_data	= pcie,
+		.size		= MSI_IRQ_NUM,
+	};
 
-	pcie->msi_domain =
-		pci_msi_create_irq_domain(dev_fwnode(dev),
-					  &advk_msi_domain_info,
-					  pcie->msi_inner_domain);
-	if (!pcie->msi_domain) {
-		irq_domain_remove(pcie->msi_inner_domain);
+	pcie->msi_inner_domain = msi_create_parent_irq_domain(&info, &advk_msi_parent_ops);
+	if (!pcie->msi_inner_domain)
 		return -ENOMEM;
-	}
 
 	return 0;
 }
 
 static void advk_pcie_remove_msi_irq_domain(struct advk_pcie *pcie)
 {
-	irq_domain_remove(pcie->msi_domain);
 	irq_domain_remove(pcie->msi_inner_domain);
 }
 
-- 
2.39.5

[PATCH 04/16] PCI: altera-msi: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Joyce Ooi <joyce.ooi@intel.com>
---
 drivers/pci/controller/Kconfig           |  1 +
 drivers/pci/controller/pcie-altera-msi.c | 44 +++++++++++-------------
 2 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 91a2d4ffc3ac4..012c18c67d9c6 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -30,6 +30,7 @@ config PCIE_ALTERA_MSI
 	tristate "Altera PCIe MSI feature"
 	depends on PCIE_ALTERA
 	depends on PCI_MSI
+	select IRQ_MSI_LIB
 	help
 	  Say Y here if you want PCIe MSI support for the Altera FPGA.
 	  This MSI driver supports Altera MSI to GIC controller IP.
diff --git a/drivers/pci/controller/pcie-altera-msi.c b/drivers/pci/controller/pcie-altera-msi.c
index a43f21eb8fbb9..2e48acd632c57 100644
--- a/drivers/pci/controller/pcie-altera-msi.c
+++ b/drivers/pci/controller/pcie-altera-msi.c
@@ -9,6 +9,7 @@
 
 #include <linux/interrupt.h>
 #include <linux/irqchip/chained_irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/init.h>
 #include <linux/module.h>
@@ -29,7 +30,6 @@ struct altera_msi {
 	DECLARE_BITMAP(used, MAX_MSI_VECTORS);
 	struct mutex		lock;	/* protect "used" bitmap */
 	struct platform_device	*pdev;
-	struct irq_domain	*msi_domain;
 	struct irq_domain	*inner_domain;
 	void __iomem		*csr_base;
 	void __iomem		*vector_base;
@@ -74,18 +74,20 @@ static void altera_msi_isr(struct irq_desc *desc)
 	chained_irq_exit(chip, desc);
 }
 
-static struct irq_chip altera_msi_irq_chip = {
-	.name = "Altera PCIe MSI",
-	.irq_mask = pci_msi_mask_irq,
-	.irq_unmask = pci_msi_unmask_irq,
-};
+#define ALTERA_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS		| \
+				   MSI_FLAG_USE_DEF_CHIP_OPS		| \
+				   MSI_FLAG_NO_AFFINITY)
 
-static struct msi_domain_info altera_msi_domain_info = {
-	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		  MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX,
-	.chip	= &altera_msi_irq_chip,
-};
+#define ALTERA_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK		| \
+				    MSI_FLAG_PCI_MSIX)
 
+static const struct msi_parent_ops altera_msi_parent_ops = {
+	.required_flags		= ALTERA_MSI_FLAGS_REQUIRED,
+	.supported_flags	= ALTERA_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.prefix			= "Altera-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
+};
 static void altera_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 {
 	struct altera_msi *msi = irq_data_get_irq_chip_data(data);
@@ -165,19 +167,16 @@ static const struct irq_domain_ops msi_domain_ops = {
 static int altera_allocate_domains(struct altera_msi *msi)
 {
 	struct fwnode_handle *fwnode = of_fwnode_handle(msi->pdev->dev.of_node);
-
-	msi->inner_domain = irq_domain_create_linear(NULL, msi->num_of_vectors,
-					     &msi_domain_ops, msi);
+	struct irq_domain_info info = {
+		.fwnode		= fwnode,
+		.ops		= &msi_domain_ops,
+		.host_data	= msi,
+		.size		= msi->num_of_vectors,
+	};
+
+	msi->inner_domain = msi_create_parent_irq_domain(&info, &altera_msi_parent_ops);
 	if (!msi->inner_domain) {
-		dev_err(&msi->pdev->dev, "failed to create IRQ domain\n");
-		return -ENOMEM;
-	}
-
-	msi->msi_domain = pci_msi_create_irq_domain(fwnode,
-				&altera_msi_domain_info, msi->inner_domain);
-	if (!msi->msi_domain) {
 		dev_err(&msi->pdev->dev, "failed to create MSI domain\n");
-		irq_domain_remove(msi->inner_domain);
 		return -ENOMEM;
 	}
 
@@ -186,7 +185,6 @@ static int altera_allocate_domains(struct altera_msi *msi)
 
 static void altera_free_domains(struct altera_msi *msi)
 {
-	irq_domain_remove(msi->msi_domain);
 	irq_domain_remove(msi->inner_domain);
 }
 
-- 
2.39.5

[PATCH 05/16] PCI: brcmstb: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Florian Fainelli <florian.fainelli@broadcom.com>
Cc: Broadcom internal kernel review list <bcm-kernel-feedback-list@broadcom.com>
Cc: Jim Quinlan <jim2101024@gmail.com>
Cc: Nicolas Saenz Julienne <nsaenz@kernel.org>
Cc: linux-rpi-kernel@lists.infradead.org
Cc: linux-arm-kernel@lists.infradead.org
---
 drivers/pci/controller/Kconfig        |  1 +
 drivers/pci/controller/pcie-brcmstb.c | 44 +++++++++++++--------------
 2 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 012c18c67d9c6..0f6cec244d4fa 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -64,6 +64,7 @@ config PCIE_BRCMSTB
 		   BMIPS_GENERIC || COMPILE_TEST
 	depends on OF
 	depends on PCI_MSI
+	select IRQ_MSI_LIB
 	default ARCH_BRCMSTB || BMIPS_GENERIC
 	help
 	  Say Y here to enable PCIe host controller support for
diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c
index 744df5bd39aea..8ea75aa13130e 100644
--- a/drivers/pci/controller/pcie-brcmstb.c
+++ b/drivers/pci/controller/pcie-brcmstb.c
@@ -12,6 +12,7 @@
 #include <linux/iopoll.h>
 #include <linux/ioport.h>
 #include <linux/irqchip/chained_irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/list.h>
@@ -269,7 +270,6 @@ struct brcm_msi {
 	struct device		*dev;
 	void __iomem		*base;
 	struct device_node	*np;
-	struct irq_domain	*msi_domain;
 	struct irq_domain	*inner_domain;
 	struct mutex		lock; /* guards the alloc/free operations */
 	u64			target_addr;
@@ -469,17 +469,20 @@ static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie,
 	writel(tmp, pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win));
 }
 
-static struct irq_chip brcm_msi_irq_chip = {
-	.name            = "BRCM STB PCIe MSI",
-	.irq_ack         = irq_chip_ack_parent,
-	.irq_mask        = pci_msi_mask_irq,
-	.irq_unmask      = pci_msi_unmask_irq,
-};
+#define BRCM_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
+				 MSI_FLAG_USE_DEF_CHIP_OPS	| \
+				 MSI_FLAG_NO_AFFINITY)
+
+#define BRCM_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK	| \
+				  MSI_FLAG_MULTI_PCI_MSI)
 
-static struct msi_domain_info brcm_msi_domain_info = {
-	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		  MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI,
-	.chip	= &brcm_msi_irq_chip,
+static const struct msi_parent_ops brcm_msi_parent_ops = {
+	.required_flags		= BRCM_MSI_FLAGS_REQUIRED,
+	.supported_flags	= BRCM_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
+	.prefix			= "BRCM-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 };
 
 static void brcm_pcie_msi_isr(struct irq_desc *desc)
@@ -588,18 +591,16 @@ static int brcm_allocate_domains(struct brcm_msi *msi)
 	struct fwnode_handle *fwnode = of_fwnode_handle(msi->np);
 	struct device *dev = msi->dev;
 
-	msi->inner_domain = irq_domain_create_linear(NULL, msi->nr, &msi_domain_ops, msi);
-	if (!msi->inner_domain) {
-		dev_err(dev, "failed to create IRQ domain\n");
-		return -ENOMEM;
-	}
+	struct irq_domain_info info = {
+		.fwnode		= fwnode,
+		.ops		= &msi_domain_ops,
+		.host_data	= msi,
+		.size		= msi->nr,
+	};
 
-	msi->msi_domain = pci_msi_create_irq_domain(fwnode,
-						    &brcm_msi_domain_info,
-						    msi->inner_domain);
-	if (!msi->msi_domain) {
+	msi->inner_domain = msi_create_parent_irq_domain(&info, &brcm_msi_parent_ops);
+	if (!msi->inner_domain) {
 		dev_err(dev, "failed to create MSI domain\n");
-		irq_domain_remove(msi->inner_domain);
 		return -ENOMEM;
 	}
 
@@ -608,7 +609,6 @@ static int brcm_allocate_domains(struct brcm_msi *msi)
 
 static void brcm_free_domains(struct brcm_msi *msi)
 {
-	irq_domain_remove(msi->msi_domain);
 	irq_domain_remove(msi->inner_domain);
 }
 
-- 
2.39.5

[PATCH 06/16] PCI: iproc: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Ray Jui <rjui@broadcom.com>
Cc: Scott Branden <sbranden@broadcom.com>
Cc: Broadcom internal kernel review list <bcm-kernel-feedback-list@broadcom.com>
Cc: linux-arm-kernel@lists.infradead.org
---
 drivers/pci/controller/Kconfig          |  1 +
 drivers/pci/controller/pcie-iproc-msi.c | 45 ++++++++++++-------------
 2 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 0f6cec244d4fa..375a019f35bd9 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -101,6 +101,7 @@ config PCIE_IPROC_MSI
 	bool "Broadcom iProc PCIe MSI support"
 	depends on PCIE_IPROC_PLATFORM || PCIE_IPROC_BCMA
 	depends on PCI_MSI
+	select IRQ_MSI_LIB
 	default ARCH_BCM_IPROC
 	help
 	  Say Y here if you want to enable MSI support for Broadcom's iProc
diff --git a/drivers/pci/controller/pcie-iproc-msi.c b/drivers/pci/controller/pcie-iproc-msi.c
index d2cb4c4f821af..d0c7f004217fb 100644
--- a/drivers/pci/controller/pcie-iproc-msi.c
+++ b/drivers/pci/controller/pcie-iproc-msi.c
@@ -5,6 +5,7 @@
 
 #include <linux/interrupt.h>
 #include <linux/irqchip/chained_irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/msi.h>
 #include <linux/of_irq.h>
@@ -101,7 +102,6 @@ struct iproc_msi {
 	struct mutex bitmap_lock;
 	unsigned int nr_msi_vecs;
 	struct irq_domain *inner_domain;
-	struct irq_domain *msi_domain;
 	unsigned int nr_eq_region;
 	unsigned int nr_msi_region;
 	void *eq_cpu;
@@ -165,16 +165,18 @@ static inline unsigned int iproc_msi_eq_offset(struct iproc_msi *msi, u32 eq)
 		return eq * EQ_LEN * sizeof(u32);
 }
 
-static struct irq_chip iproc_msi_irq_chip = {
-	.name = "iProc-MSI",
+#define IPROC_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
+				  MSI_FLAG_USE_DEF_CHIP_OPS)
+#define IPROC_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK	| \
+				   MSI_FLAG_PCI_MSIX)
+
+static struct msi_parent_ops iproc_msi_parent_ops = {
+	.required_flags		= IPROC_MSI_FLAGS_REQUIRED,
+	.supported_flags	= IPROC_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.prefix			= "iProc-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 };
-
-static struct msi_domain_info iproc_msi_domain_info = {
-	.flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		MSI_FLAG_PCI_MSIX,
-	.chip = &iproc_msi_irq_chip,
-};
-
 /*
  * In iProc PCIe core, each MSI group is serviced by a GIC interrupt and a
  * dedicated event queue.  Each MSI group can support up to 64 MSI vectors.
@@ -446,27 +448,22 @@ static void iproc_msi_disable(struct iproc_msi *msi)
 static int iproc_msi_alloc_domains(struct device_node *node,
 				   struct iproc_msi *msi)
 {
-	msi->inner_domain = irq_domain_create_linear(NULL, msi->nr_msi_vecs,
-						     &msi_domain_ops, msi);
+	struct irq_domain_info info = {
+		.fwnode		= of_fwnode_handle(node),
+		.ops		= &msi_domain_ops,
+		.host_data	= msi,
+		.size		= msi->nr_msi_vecs,
+	};
+
+	msi->inner_domain = msi_create_parent_irq_domain(&info, &iproc_msi_parent_ops);
 	if (!msi->inner_domain)
 		return -ENOMEM;
 
-	msi->msi_domain = pci_msi_create_irq_domain(of_fwnode_handle(node),
-						    &iproc_msi_domain_info,
-						    msi->inner_domain);
-	if (!msi->msi_domain) {
-		irq_domain_remove(msi->inner_domain);
-		return -ENOMEM;
-	}
-
 	return 0;
 }
 
 static void iproc_msi_free_domains(struct iproc_msi *msi)
 {
-	if (msi->msi_domain)
-		irq_domain_remove(msi->msi_domain);
-
 	if (msi->inner_domain)
 		irq_domain_remove(msi->inner_domain);
 }
@@ -542,7 +539,7 @@ int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node)
 	msi->nr_cpus = num_possible_cpus();
 
 	if (msi->nr_cpus == 1)
-		iproc_msi_domain_info.flags |=  MSI_FLAG_MULTI_PCI_MSI;
+		iproc_msi_parent_ops.supported_flags |= MSI_FLAG_MULTI_PCI_MSI;
 
 	msi->nr_irqs = of_irq_count(node);
 	if (!msi->nr_irqs) {
-- 
2.39.5

[PATCH 07/16] PCI: mediatek-gen3: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Ryder Lee <ryder.lee@mediatek.com>
Cc: Jianjun Wang <jianjun.wang@mediatek.com>
Cc: Matthias Brugger <matthias.bgg@gmail.com>
Cc: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
Cc: linux-mediatek@lists.infradead.org
Cc: linux-arm-kernel@lists.infradead.org
---
 drivers/pci/controller/Kconfig              |  1 +
 drivers/pci/controller/pcie-mediatek-gen3.c | 67 ++++++++-------------
 2 files changed, 26 insertions(+), 42 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 375a019f35bd9..ec32c343a751d 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -203,6 +203,7 @@ config PCIE_MEDIATEK_GEN3
 	tristate "MediaTek Gen3 PCIe controller"
 	depends on ARCH_AIROHA || ARCH_MEDIATEK || COMPILE_TEST
 	depends on PCI_MSI
+	select IRQ_MSI_LIB
 	help
 	  Adds support for PCIe Gen3 MAC controller for MediaTek SoCs.
 	  This PCIe controller is compatible with Gen3, Gen2 and Gen1 speed,
diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c
index 5464b4ae5c20c..97147f43e41c5 100644
--- a/drivers/pci/controller/pcie-mediatek-gen3.c
+++ b/drivers/pci/controller/pcie-mediatek-gen3.c
@@ -12,6 +12,7 @@
 #include <linux/delay.h>
 #include <linux/iopoll.h>
 #include <linux/irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
@@ -187,7 +188,6 @@ struct mtk_msi_set {
  * @saved_irq_state: IRQ enable state saved at suspend time
  * @irq_lock: lock protecting IRQ register access
  * @intx_domain: legacy INTx IRQ domain
- * @msi_domain: MSI IRQ domain
  * @msi_bottom_domain: MSI IRQ bottom domain
  * @msi_sets: MSI sets information
  * @lock: lock protecting IRQ bit map
@@ -210,7 +210,6 @@ struct mtk_gen3_pcie {
 	u32 saved_irq_state;
 	raw_spinlock_t irq_lock;
 	struct irq_domain *intx_domain;
-	struct irq_domain *msi_domain;
 	struct irq_domain *msi_bottom_domain;
 	struct mtk_msi_set msi_sets[PCIE_MSI_SET_NUM];
 	struct mutex lock;
@@ -526,30 +525,22 @@ static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie)
 	return 0;
 }
 
-static void mtk_pcie_msi_irq_mask(struct irq_data *data)
-{
-	pci_msi_mask_irq(data);
-	irq_chip_mask_parent(data);
-}
-
-static void mtk_pcie_msi_irq_unmask(struct irq_data *data)
-{
-	pci_msi_unmask_irq(data);
-	irq_chip_unmask_parent(data);
-}
-
-static struct irq_chip mtk_msi_irq_chip = {
-	.irq_ack = irq_chip_ack_parent,
-	.irq_mask = mtk_pcie_msi_irq_mask,
-	.irq_unmask = mtk_pcie_msi_irq_unmask,
-	.name = "MSI",
-};
-
-static struct msi_domain_info mtk_msi_domain_info = {
-	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		  MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX |
-		  MSI_FLAG_MULTI_PCI_MSI,
-	.chip	= &mtk_msi_irq_chip,
+#define MTK_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
+				MSI_FLAG_USE_DEF_CHIP_OPS	| \
+				MSI_FLAG_NO_AFFINITY		| \
+				MSI_FLAG_PCI_MSI_MASK_PARENT)
+
+#define MTK_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK		| \
+				 MSI_FLAG_PCI_MSIX		| \
+				 MSI_FLAG_MULTI_PCI_MSI)
+
+static const struct msi_parent_ops mtk_msi_parent_ops = {
+	.required_flags		= MTK_MSI_FLAGS_REQUIRED,
+	.supported_flags	= MTK_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
+	.prefix			= "MTK3-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 };
 
 static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
@@ -756,28 +747,23 @@ static int mtk_pcie_init_irq_domains(struct mtk_gen3_pcie *pcie)
 	/* Setup MSI */
 	mutex_init(&pcie->lock);
 
-	pcie->msi_bottom_domain = irq_domain_create_linear(dev_fwnode(dev), PCIE_MSI_IRQS_NUM,
-							   &mtk_msi_bottom_domain_ops, pcie);
+	struct irq_domain_info info = {
+		.fwnode		= dev_fwnode(dev),
+		.ops		= &mtk_msi_bottom_domain_ops,
+		.host_data	= pcie,
+		.size		= PCIE_MSI_IRQS_NUM,
+	};
+
+	pcie->msi_bottom_domain = msi_create_parent_irq_domain(&info, &mtk_msi_parent_ops);
 	if (!pcie->msi_bottom_domain) {
 		dev_err(dev, "failed to create MSI bottom domain\n");
 		ret = -ENODEV;
 		goto err_msi_bottom_domain;
 	}
 
-	pcie->msi_domain = pci_msi_create_irq_domain(dev->fwnode,
-						     &mtk_msi_domain_info,
-						     pcie->msi_bottom_domain);
-	if (!pcie->msi_domain) {
-		dev_err(dev, "failed to create MSI domain\n");
-		ret = -ENODEV;
-		goto err_msi_domain;
-	}
-
 	of_node_put(intc_node);
 	return 0;
 
-err_msi_domain:
-	irq_domain_remove(pcie->msi_bottom_domain);
 err_msi_bottom_domain:
 	irq_domain_remove(pcie->intx_domain);
 out_put_node:
@@ -792,9 +778,6 @@ static void mtk_pcie_irq_teardown(struct mtk_gen3_pcie *pcie)
 	if (pcie->intx_domain)
 		irq_domain_remove(pcie->intx_domain);
 
-	if (pcie->msi_domain)
-		irq_domain_remove(pcie->msi_domain);
-
 	if (pcie->msi_bottom_domain)
 		irq_domain_remove(pcie->msi_bottom_domain);
 
-- 
2.39.5

[PATCH 08/16] PCI: mediatek: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Ryder Lee <ryder.lee@mediatek.com>
Cc: Jianjun Wang <jianjun.wang@mediatek.com>
Cc: Matthias Brugger <matthias.bgg@gmail.com>
Cc: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
Cc: linux-mediatek@lists.infradead.org
Cc: linux-arm-kernel@lists.infradead.org
---
 drivers/pci/controller/Kconfig         |  1 +
 drivers/pci/controller/pcie-mediatek.c | 46 ++++++++++++--------------
 2 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index ec32c343a751d..65289a171333c 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -195,6 +195,7 @@ config PCIE_MEDIATEK
 	depends on ARCH_AIROHA || ARCH_MEDIATEK || COMPILE_TEST
 	depends on OF
 	depends on PCI_MSI
+	select IRQ_MSI_LIB
 	help
 	  Say Y here if you want to enable PCIe controller support on
 	  MediaTek SoCs.
diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c
index e1934aa06c8d5..3ac5d14dd543e 100644
--- a/drivers/pci/controller/pcie-mediatek.c
+++ b/drivers/pci/controller/pcie-mediatek.c
@@ -12,6 +12,7 @@
 #include <linux/iopoll.h>
 #include <linux/irq.h>
 #include <linux/irqchip/chained_irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/mfd/syscon.h>
@@ -180,7 +181,6 @@ struct mtk_pcie_soc {
  * @irq: GIC irq
  * @irq_domain: legacy INTx IRQ domain
  * @inner_domain: inner IRQ domain
- * @msi_domain: MSI IRQ domain
  * @lock: protect the msi_irq_in_use bitmap
  * @msi_irq_in_use: bit map for assigned MSI IRQ
  */
@@ -200,7 +200,6 @@ struct mtk_pcie_port {
 	int irq;
 	struct irq_domain *irq_domain;
 	struct irq_domain *inner_domain;
-	struct irq_domain *msi_domain;
 	struct mutex lock;
 	DECLARE_BITMAP(msi_irq_in_use, MTK_MSI_IRQS_NUM);
 };
@@ -470,17 +469,20 @@ static const struct irq_domain_ops msi_domain_ops = {
 	.free	= mtk_pcie_irq_domain_free,
 };
 
-static struct irq_chip mtk_msi_irq_chip = {
-	.name		= "MTK PCIe MSI",
-	.irq_ack	= irq_chip_ack_parent,
-	.irq_mask	= pci_msi_mask_irq,
-	.irq_unmask	= pci_msi_unmask_irq,
-};
+#define MTK_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
+				MSI_FLAG_USE_DEF_CHIP_OPS	| \
+				MSI_FLAG_NO_AFFINITY)
+
+#define MTK_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK		| \
+				 MSI_FLAG_PCI_MSIX)
 
-static struct msi_domain_info mtk_msi_domain_info = {
-	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		  MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX,
-	.chip	= &mtk_msi_irq_chip,
+static const struct msi_parent_ops mtk_msi_parent_ops = {
+	.required_flags		= MTK_MSI_FLAGS_REQUIRED,
+	.supported_flags	= MTK_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
+	.prefix			= "MTK-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 };
 
 static int mtk_pcie_allocate_msi_domains(struct mtk_pcie_port *port)
@@ -489,21 +491,19 @@ static int mtk_pcie_allocate_msi_domains(struct mtk_pcie_port *port)
 
 	mutex_init(&port->lock);
 
-	port->inner_domain = irq_domain_create_linear(fwnode, MTK_MSI_IRQS_NUM,
-						      &msi_domain_ops, port);
+	struct irq_domain_info info = {
+		.fwnode		= fwnode,
+		.ops		= &msi_domain_ops,
+		.host_data	= port,
+		.size		= MTK_MSI_IRQS_NUM,
+	};
+
+	port->inner_domain = msi_create_parent_irq_domain(&info, &mtk_msi_parent_ops);
 	if (!port->inner_domain) {
 		dev_err(port->pcie->dev, "failed to create IRQ domain\n");
 		return -ENOMEM;
 	}
 
-	port->msi_domain = pci_msi_create_irq_domain(fwnode, &mtk_msi_domain_info,
-						     port->inner_domain);
-	if (!port->msi_domain) {
-		dev_err(port->pcie->dev, "failed to create MSI domain\n");
-		irq_domain_remove(port->inner_domain);
-		return -ENOMEM;
-	}
-
 	return 0;
 }
 
@@ -532,8 +532,6 @@ static void mtk_pcie_irq_teardown(struct mtk_pcie *pcie)
 			irq_domain_remove(port->irq_domain);
 
 		if (IS_ENABLED(CONFIG_PCI_MSI)) {
-			if (port->msi_domain)
-				irq_domain_remove(port->msi_domain);
 			if (port->inner_domain)
 				irq_domain_remove(port->inner_domain);
 		}
-- 
2.39.5

[PATCH 09/16] PCI: rcar-host: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Marek Vasut <marek.vasut+renesas@gmail.com>
Cc: Yoshihiro Shimoda <yoshihiro.shimoda.uh@renesas.com>
Cc: Geert Uytterhoeven <geert+renesas@glider.be>
Cc: Magnus Damm <magnus.damm@gmail.com>
Cc: linux-renesas-soc@vger.kernel.org
---
 drivers/pci/controller/Kconfig          |  1 +
 drivers/pci/controller/pcie-rcar-host.c | 69 +++++++++----------------
 2 files changed, 26 insertions(+), 44 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 65289a171333c..8b9492e9ae693 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -243,6 +243,7 @@ config PCIE_RCAR_HOST
 	bool "Renesas R-Car PCIe controller (host mode)"
 	depends on ARCH_RENESAS || COMPILE_TEST
 	depends on PCI_MSI
+	select IRQ_MSI_LIB
 	help
 	  Say Y here if you want PCIe controller support on R-Car SoCs in host
 	  mode.
diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c
index c32b803a47c7c..ad2bda635d47f 100644
--- a/drivers/pci/controller/pcie-rcar-host.c
+++ b/drivers/pci/controller/pcie-rcar-host.c
@@ -17,6 +17,7 @@
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -597,30 +598,6 @@ static irqreturn_t rcar_pcie_msi_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static void rcar_msi_top_irq_ack(struct irq_data *d)
-{
-	irq_chip_ack_parent(d);
-}
-
-static void rcar_msi_top_irq_mask(struct irq_data *d)
-{
-	pci_msi_mask_irq(d);
-	irq_chip_mask_parent(d);
-}
-
-static void rcar_msi_top_irq_unmask(struct irq_data *d)
-{
-	pci_msi_unmask_irq(d);
-	irq_chip_unmask_parent(d);
-}
-
-static struct irq_chip rcar_msi_top_chip = {
-	.name		= "PCIe MSI",
-	.irq_ack	= rcar_msi_top_irq_ack,
-	.irq_mask	= rcar_msi_top_irq_mask,
-	.irq_unmask	= rcar_msi_top_irq_unmask,
-};
-
 static void rcar_msi_irq_ack(struct irq_data *d)
 {
 	struct rcar_msi *msi = irq_data_get_irq_chip_data(d);
@@ -718,30 +695,37 @@ static const struct irq_domain_ops rcar_msi_domain_ops = {
 	.free	= rcar_msi_domain_free,
 };
 
-static struct msi_domain_info rcar_msi_info = {
-	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		  MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI,
-	.chip	= &rcar_msi_top_chip,
+#define RCAR_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
+				 MSI_FLAG_USE_DEF_CHIP_OPS	| \
+				 MSI_FLAG_PCI_MSI_MASK_PARENT	| \
+				 MSI_FLAG_NO_AFFINITY)
+
+#define RCAR_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK	| \
+				  MSI_FLAG_MULTI_PCI_MSI)
+
+static const struct msi_parent_ops rcar_msi_parent_ops = {
+	.required_flags		= RCAR_MSI_FLAGS_REQUIRED,
+	.supported_flags	= RCAR_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
+	.prefix			= "RCAR-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 };
 
 static int rcar_allocate_domains(struct rcar_msi *msi)
 {
 	struct rcar_pcie *pcie = &msi_to_host(msi)->pcie;
 	struct fwnode_handle *fwnode = dev_fwnode(pcie->dev);
-	struct irq_domain *parent;
-
-	parent = irq_domain_create_linear(fwnode, INT_PCI_MSI_NR,
-					  &rcar_msi_domain_ops, msi);
-	if (!parent) {
-		dev_err(pcie->dev, "failed to create IRQ domain\n");
-		return -ENOMEM;
-	}
-	irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS);
-
-	msi->domain = pci_msi_create_irq_domain(fwnode, &rcar_msi_info, parent);
+	struct irq_domain_info info = {
+		.fwnode		= fwnode,
+		.ops		= &rcar_msi_domain_ops,
+		.host_data	= msi,
+		.size		= INT_PCI_MSI_NR,
+	};
+
+	msi->domain = msi_create_parent_irq_domain(&info, &rcar_msi_parent_ops);
 	if (!msi->domain) {
-		dev_err(pcie->dev, "failed to create MSI domain\n");
-		irq_domain_remove(parent);
+		dev_err(pcie->dev, "failed to create IRQ domain\n");
 		return -ENOMEM;
 	}
 
@@ -750,10 +734,7 @@ static int rcar_allocate_domains(struct rcar_msi *msi)
 
 static void rcar_free_domains(struct rcar_msi *msi)
 {
-	struct irq_domain *parent = msi->domain->parent;
-
 	irq_domain_remove(msi->domain);
-	irq_domain_remove(parent);
 }
 
 static int rcar_pcie_enable_msi(struct rcar_pcie_host *host)
-- 
2.39.5

[PATCH 10/16] PCI: xilinx-xdma: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Michal Simek <michal.simek@amd.com>
Cc: linux-arm-kernel@lists.infradead.org
---
 drivers/pci/controller/Kconfig              |  1 +
 drivers/pci/controller/pcie-xilinx-dma-pl.c | 48 +++++++++------------
 2 files changed, 22 insertions(+), 27 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 8b9492e9ae693..c9b6180239732 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -331,6 +331,7 @@ config PCIE_XILINX_DMA_PL
 	depends on ARCH_ZYNQMP || COMPILE_TEST
 	depends on PCI_MSI
 	select PCI_HOST_COMMON
+	select IRQ_MSI_LIB
 	help
 	  Say 'Y' here if you want kernel support for the Xilinx PL DMA
 	  PCIe host bridge. The controller is a Soft IP which can act as
diff --git a/drivers/pci/controller/pcie-xilinx-dma-pl.c b/drivers/pci/controller/pcie-xilinx-dma-pl.c
index dc9690a535e16..fbc379fd118b4 100644
--- a/drivers/pci/controller/pcie-xilinx-dma-pl.c
+++ b/drivers/pci/controller/pcie-xilinx-dma-pl.c
@@ -7,6 +7,7 @@
 #include <linux/bitfield.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -90,7 +91,6 @@ struct xilinx_pl_dma_variant {
 };
 
 struct xilinx_msi {
-	struct irq_domain	*msi_domain;
 	unsigned long		*bitmap;
 	struct irq_domain	*dev_domain;
 	struct mutex		lock;		/* Protect bitmap variable */
@@ -373,20 +373,20 @@ static irqreturn_t xilinx_pl_dma_pcie_intr_handler(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
-static struct irq_chip xilinx_msi_irq_chip = {
-	.name = "pl_dma:PCIe MSI",
-	.irq_enable = pci_msi_unmask_irq,
-	.irq_disable = pci_msi_mask_irq,
-	.irq_mask = pci_msi_mask_irq,
-	.irq_unmask = pci_msi_unmask_irq,
-};
+#define XILINX_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
+				   MSI_FLAG_USE_DEF_CHIP_OPS	| \
+				   MSI_FLAG_NO_AFFINITY)
 
-static struct msi_domain_info xilinx_msi_domain_info = {
-	.flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		 MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI,
-	.chip = &xilinx_msi_irq_chip,
-};
+#define XILINX_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK	| \
+				    MSI_FLAG_MULTI_PCI_MSI)
 
+static const struct msi_parent_ops xilinx_msi_parent_ops = {
+	.required_flags		= XILINX_MSI_FLAGS_REQUIRED,
+	.supported_flags	= XILINX_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.prefix			= "pl_dma-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
+};
 static void xilinx_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 {
 	struct pl_dma_pcie *pcie = irq_data_get_irq_chip_data(data);
@@ -458,11 +458,6 @@ static void xilinx_pl_dma_pcie_free_irq_domains(struct pl_dma_pcie *port)
 		irq_domain_remove(msi->dev_domain);
 		msi->dev_domain = NULL;
 	}
-
-	if (msi->msi_domain) {
-		irq_domain_remove(msi->msi_domain);
-		msi->msi_domain = NULL;
-	}
 }
 
 static int xilinx_pl_dma_pcie_init_msi_irq_domain(struct pl_dma_pcie *port)
@@ -471,18 +466,17 @@ static int xilinx_pl_dma_pcie_init_msi_irq_domain(struct pl_dma_pcie *port)
 	struct xilinx_msi *msi = &port->msi;
 	int size = BITS_TO_LONGS(XILINX_NUM_MSI_IRQS) * sizeof(long);
 	struct fwnode_handle *fwnode = of_fwnode_handle(port->dev->of_node);
-
-	msi->dev_domain = irq_domain_create_linear(NULL, XILINX_NUM_MSI_IRQS,
-						   &dev_msi_domain_ops, port);
+	struct irq_domain_info info = {
+		.fwnode		= fwnode,
+		.ops		= &dev_msi_domain_ops,
+		.host_data	= port,
+		.size		= XILINX_NUM_MSI_IRQS,
+	};
+
+	msi->dev_domain  = msi_create_parent_irq_domain(&info, &xilinx_msi_parent_ops);
 	if (!msi->dev_domain)
 		goto out;
 
-	msi->msi_domain = pci_msi_create_irq_domain(fwnode,
-						    &xilinx_msi_domain_info,
-						    msi->dev_domain);
-	if (!msi->msi_domain)
-		goto out;
-
 	mutex_init(&msi->lock);
 	msi->bitmap = kzalloc(size, GFP_KERNEL);
 	if (!msi->bitmap)
-- 
2.39.5

[PATCH 11/16] PCI: xilinx-nwl: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Michal Simek <michal.simek@amd.com>
Cc: linux-arm-kernel@lists.infradead.org
---
 drivers/pci/controller/Kconfig           |  1 +
 drivers/pci/controller/pcie-xilinx-nwl.c | 45 ++++++++++++------------
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index c9b6180239732..118ff622fa69e 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -342,6 +342,7 @@ config PCIE_XILINX_NWL
 	bool "Xilinx NWL PCIe controller"
 	depends on ARCH_ZYNQMP || COMPILE_TEST
 	depends on PCI_MSI
+	select IRQ_MSI_LIB
 	help
 	 Say 'Y' here if you want kernel support for Xilinx
 	 NWL PCIe controller. The controller can act as Root Port
diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c
index c8b05477b7198..de76c836915f0 100644
--- a/drivers/pci/controller/pcie-xilinx-nwl.c
+++ b/drivers/pci/controller/pcie-xilinx-nwl.c
@@ -10,6 +10,7 @@
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -145,7 +146,6 @@
 #define LINK_WAIT_USLEEP_MAX           100000
 
 struct nwl_msi {			/* MSI information */
-	struct irq_domain *msi_domain;
 	DECLARE_BITMAP(bitmap, INT_PCI_MSI_NR);
 	struct irq_domain *dev_domain;
 	struct mutex lock;		/* protect bitmap variable */
@@ -418,19 +418,22 @@ static const struct irq_domain_ops intx_domain_ops = {
 };
 
 #ifdef CONFIG_PCI_MSI
-static struct irq_chip nwl_msi_irq_chip = {
-	.name = "nwl_pcie:msi",
-	.irq_enable = pci_msi_unmask_irq,
-	.irq_disable = pci_msi_mask_irq,
-	.irq_mask = pci_msi_mask_irq,
-	.irq_unmask = pci_msi_unmask_irq,
-};
 
-static struct msi_domain_info nwl_msi_domain_info = {
-	.flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		 MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI,
-	.chip = &nwl_msi_irq_chip,
+#define NWL_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
+				MSI_FLAG_USE_DEF_CHIP_OPS	| \
+				MSI_FLAG_NO_AFFINITY)
+
+#define NWL_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK		| \
+				 MSI_FLAG_MULTI_PCI_MSI)
+
+static const struct msi_parent_ops nwl_msi_parent_ops = {
+	.required_flags		= NWL_MSI_FLAGS_REQUIRED,
+	.supported_flags	= NWL_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.prefix			= "nwl-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 };
+
 #endif
 
 static void nwl_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
@@ -497,20 +500,18 @@ static int nwl_pcie_init_msi_irq_domain(struct nwl_pcie *pcie)
 	struct device *dev = pcie->dev;
 	struct fwnode_handle *fwnode = of_fwnode_handle(dev->of_node);
 	struct nwl_msi *msi = &pcie->msi;
-
-	msi->dev_domain = irq_domain_create_linear(NULL, INT_PCI_MSI_NR, &dev_msi_domain_ops, pcie);
+	struct irq_domain_info info = {
+		.fwnode		= fwnode,
+		.ops		= &dev_msi_domain_ops,
+		.host_data	= pcie,
+		.size		= INT_PCI_MSI_NR,
+	};
+
+	msi->dev_domain  = msi_create_parent_irq_domain(&info, &nwl_msi_parent_ops);
 	if (!msi->dev_domain) {
 		dev_err(dev, "failed to create dev IRQ domain\n");
 		return -ENOMEM;
 	}
-	msi->msi_domain = pci_msi_create_irq_domain(fwnode,
-						    &nwl_msi_domain_info,
-						    msi->dev_domain);
-	if (!msi->msi_domain) {
-		dev_err(dev, "failed to create msi IRQ domain\n");
-		irq_domain_remove(msi->dev_domain);
-		return -ENOMEM;
-	}
 #endif
 	return 0;
 }
-- 
2.39.5

[PATCH 12/16] PCI: xilinx: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Michal Simek <michal.simek@amd.com>
Cc: linux-arm-kernel@lists.infradead.org
---
 drivers/pci/controller/Kconfig       |  1 +
 drivers/pci/controller/pcie-xilinx.c | 55 ++++++++++++++++------------
 2 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 118ff622fa69e..8f56ffd029ba2 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -322,6 +322,7 @@ config PCIE_XILINX
 	bool "Xilinx AXI PCIe controller"
 	depends on OF
 	depends on PCI_MSI
+	select IRQ_MSI_LIB
 	help
 	  Say 'Y' here if you want kernel to support the Xilinx AXI PCIe
 	  Host Bridge driver.
diff --git a/drivers/pci/controller/pcie-xilinx.c b/drivers/pci/controller/pcie-xilinx.c
index e36aa874bae92..ddc2ab6c48b9b 100644
--- a/drivers/pci/controller/pcie-xilinx.c
+++ b/drivers/pci/controller/pcie-xilinx.c
@@ -12,6 +12,7 @@
 
 #include <linux/interrupt.h>
 #include <linux/irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -203,11 +204,6 @@ static void xilinx_msi_top_irq_ack(struct irq_data *d)
 	 */
 }
 
-static struct irq_chip xilinx_msi_top_chip = {
-	.name		= "PCIe MSI",
-	.irq_ack	= xilinx_msi_top_irq_ack,
-};
-
 static void xilinx_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 {
 	struct xilinx_pcie *pcie = irq_data_get_irq_chip_data(data);
@@ -264,29 +260,43 @@ static const struct irq_domain_ops xilinx_msi_domain_ops = {
 	.free	= xilinx_msi_domain_free,
 };
 
-static struct msi_domain_info xilinx_msi_info = {
-	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		  MSI_FLAG_NO_AFFINITY,
-	.chip	= &xilinx_msi_top_chip,
+static bool xilinx_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
+				     struct irq_domain *real_parent, struct msi_domain_info *info)
+{
+	struct irq_chip *chip = info->chip;
+
+	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
+		return false;
+
+	chip->irq_ack = xilinx_msi_top_irq_ack;
+	return true;
+}
+
+#define XILINX_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
+				   MSI_FLAG_USE_DEF_CHIP_OPS	| \
+				   MSI_FLAG_NO_AFFINITY)
+
+static const struct msi_parent_ops xilinx_msi_parent_ops = {
+	.required_flags		= XILINX_MSI_FLAGS_REQUIRED,
+	.supported_flags	= MSI_GENERIC_FLAGS_MASK,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.prefix			= "xilinx-",
+	.init_dev_msi_info	= xilinx_init_dev_msi_info,
 };
 
 static int xilinx_allocate_msi_domains(struct xilinx_pcie *pcie)
 {
 	struct fwnode_handle *fwnode = dev_fwnode(pcie->dev);
-	struct irq_domain *parent;
-
-	parent = irq_domain_create_linear(fwnode, XILINX_NUM_MSI_IRQS,
-					  &xilinx_msi_domain_ops, pcie);
-	if (!parent) {
-		dev_err(pcie->dev, "failed to create IRQ domain\n");
-		return -ENOMEM;
-	}
-	irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS);
-
-	pcie->msi_domain = pci_msi_create_irq_domain(fwnode, &xilinx_msi_info, parent);
+	struct irq_domain_info info = {
+		.fwnode		= fwnode,
+		.ops		= &xilinx_msi_domain_ops,
+		.host_data	= pcie,
+		.size		= XILINX_NUM_MSI_IRQS,
+	};
+
+	pcie->msi_domain = msi_create_parent_irq_domain(&info, &xilinx_msi_parent_ops);
 	if (!pcie->msi_domain) {
 		dev_err(pcie->dev, "failed to create MSI domain\n");
-		irq_domain_remove(parent);
 		return -ENOMEM;
 	}
 
@@ -295,10 +305,7 @@ static int xilinx_allocate_msi_domains(struct xilinx_pcie *pcie)
 
 static void xilinx_free_msi_domains(struct xilinx_pcie *pcie)
 {
-	struct irq_domain *parent = pcie->msi_domain->parent;
-
 	irq_domain_remove(pcie->msi_domain);
-	irq_domain_remove(parent);
 }
 
 /* INTx Functions */
-- 
2.39.5

[PATCH 13/16] PCI: plda: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Daire McNamara <daire.mcnamara@microchip.com>
---
 drivers/pci/controller/plda/Kconfig          |  1 +
 drivers/pci/controller/plda/pcie-plda-host.c | 44 ++++++++++----------
 drivers/pci/controller/plda/pcie-plda.h      |  1 -
 3 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/drivers/pci/controller/plda/Kconfig b/drivers/pci/controller/plda/Kconfig
index c0e14146d7e45..62120101139cb 100644
--- a/drivers/pci/controller/plda/Kconfig
+++ b/drivers/pci/controller/plda/Kconfig
@@ -5,6 +5,7 @@ menu "PLDA-based PCIe controllers"
 
 config PCIE_PLDA_HOST
 	bool
+	select IRQ_MSI_LIB
 
 config PCIE_MICROCHIP_HOST
 	tristate "Microchip AXI PCIe controller"
diff --git a/drivers/pci/controller/plda/pcie-plda-host.c b/drivers/pci/controller/plda/pcie-plda-host.c
index 3abedf723215c..92866840875e3 100644
--- a/drivers/pci/controller/plda/pcie-plda-host.c
+++ b/drivers/pci/controller/plda/pcie-plda-host.c
@@ -11,6 +11,7 @@
 #include <linux/align.h>
 #include <linux/bitfield.h>
 #include <linux/irqchip/chained_irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/irqdomain.h>
 #include <linux/msi.h>
 #include <linux/pci_regs.h>
@@ -134,17 +135,19 @@ static const struct irq_domain_ops msi_domain_ops = {
 	.free	= plda_irq_msi_domain_free,
 };
 
-static struct irq_chip plda_msi_irq_chip = {
-	.name = "PLDA PCIe MSI",
-	.irq_ack = irq_chip_ack_parent,
-	.irq_mask = pci_msi_mask_irq,
-	.irq_unmask = pci_msi_unmask_irq,
-};
-
-static struct msi_domain_info plda_msi_domain_info = {
-	.flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-		 MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX,
-	.chip = &plda_msi_irq_chip,
+#define PLDA_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
+				 MSI_FLAG_USE_DEF_CHIP_OPS	| \
+				 MSI_FLAG_NO_AFFINITY)
+#define PLDA_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK	| \
+				  MSI_FLAG_PCI_MSIX)
+
+static const struct msi_parent_ops plda_msi_parent_ops = {
+	.required_flags		= PLDA_MSI_FLAGS_REQUIRED,
+	.supported_flags	= PLDA_MSI_FLAGS_SUPPORTED,
+	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+	.prefix			= "PLDA-",
+	.init_dev_msi_info	= msi_lib_init_dev_msi_info,
 };
 
 static int plda_allocate_msi_domains(struct plda_pcie_rp *port)
@@ -155,21 +158,19 @@ static int plda_allocate_msi_domains(struct plda_pcie_rp *port)
 
 	mutex_init(&port->msi.lock);
 
-	msi->dev_domain = irq_domain_create_linear(NULL, msi->num_vectors, &msi_domain_ops, port);
+	struct irq_domain_info info = {
+		.fwnode		= fwnode,
+		.ops		= &msi_domain_ops,
+		.host_data	= port,
+		.size		= msi->num_vectors,
+	};
+
+	msi->dev_domain = msi_create_parent_irq_domain(&info, &plda_msi_parent_ops);
 	if (!msi->dev_domain) {
 		dev_err(dev, "failed to create IRQ domain\n");
 		return -ENOMEM;
 	}
 
-	msi->msi_domain = pci_msi_create_irq_domain(fwnode,
-						    &plda_msi_domain_info,
-						    msi->dev_domain);
-	if (!msi->msi_domain) {
-		dev_err(dev, "failed to create MSI domain\n");
-		irq_domain_remove(msi->dev_domain);
-		return -ENOMEM;
-	}
-
 	return 0;
 }
 
@@ -563,7 +564,6 @@ static void plda_pcie_irq_domain_deinit(struct plda_pcie_rp *pcie)
 	irq_set_chained_handler_and_data(pcie->msi_irq, NULL, NULL);
 	irq_set_chained_handler_and_data(pcie->intx_irq, NULL, NULL);
 
-	irq_domain_remove(pcie->msi.msi_domain);
 	irq_domain_remove(pcie->msi.dev_domain);
 
 	irq_domain_remove(pcie->intx_domain);
diff --git a/drivers/pci/controller/plda/pcie-plda.h b/drivers/pci/controller/plda/pcie-plda.h
index 61ece26065ea0..6b8665df7bf0f 100644
--- a/drivers/pci/controller/plda/pcie-plda.h
+++ b/drivers/pci/controller/plda/pcie-plda.h
@@ -164,7 +164,6 @@ struct plda_pcie_host_ops {
 
 struct plda_msi {
 	struct mutex lock;		/* Protect used bitmap */
-	struct irq_domain *msi_domain;
 	struct irq_domain *dev_domain;
 	u32 num_vectors;
 	u64 vector_phy;
-- 
2.39.5

[PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: K. Y. Srinivasan <kys@microsoft.com>
Cc: Haiyang Zhang <haiyangz@microsoft.com>
Cc: Wei Liu <wei.liu@kernel.org>
Cc: Dexuan Cui <decui@microsoft.com>
Cc: linux-hyperv@vger.kernel.org
---
 drivers/pci/Kconfig                 |  1 +
 drivers/pci/controller/pci-hyperv.c | 98 +++++++++++++++++++++++------
 2 files changed, 80 insertions(+), 19 deletions(-)

diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
index 9c0e4aaf4e8cb..9a249c65aedcd 100644
--- a/drivers/pci/Kconfig
+++ b/drivers/pci/Kconfig
@@ -223,6 +223,7 @@ config PCI_HYPERV
 	tristate "Hyper-V PCI Frontend"
 	depends on ((X86 && X86_64) || ARM64) && HYPERV && PCI_MSI && SYSFS
 	select PCI_HYPERV_INTERFACE
+	select IRQ_MSI_LIB
 	help
 	  The PCI device frontend driver allows the kernel to import arbitrary
 	  PCI devices from a PCI backend to support PCI driver domains.
diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
index ef5d655a0052c..3a24fadddb83b 100644
--- a/drivers/pci/controller/pci-hyperv.c
+++ b/drivers/pci/controller/pci-hyperv.c
@@ -44,6 +44,7 @@
 #include <linux/delay.h>
 #include <linux/semaphore.h>
 #include <linux/irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/msi.h>
 #include <linux/hyperv.h>
 #include <linux/refcount.h>
@@ -508,7 +509,6 @@ struct hv_pcibus_device {
 	struct list_head children;
 	struct list_head dr_list;
 
-	struct msi_domain_info msi_info;
 	struct irq_domain *irq_domain;
 
 	struct workqueue_struct *wq;
@@ -1687,7 +1687,7 @@ static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
 	struct msi_desc *msi = irq_data_get_msi_desc(irq_data);
 
 	pdev = msi_desc_to_pci_dev(msi);
-	hbus = info->data;
+	hbus = domain->host_data;
 	int_desc = irq_data_get_irq_chip_data(irq_data);
 	if (!int_desc)
 		return;
@@ -1705,7 +1705,6 @@ static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
 
 static void hv_irq_mask(struct irq_data *data)
 {
-	pci_msi_mask_irq(data);
 	if (data->parent_data->chip->irq_mask)
 		irq_chip_mask_parent(data);
 }
@@ -1716,7 +1715,6 @@ static void hv_irq_unmask(struct irq_data *data)
 
 	if (data->parent_data->chip->irq_unmask)
 		irq_chip_unmask_parent(data);
-	pci_msi_unmask_irq(data);
 }
 
 struct compose_comp_ctxt {
@@ -2101,6 +2099,44 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 	msg->data = 0;
 }
 
+static bool hv_pcie_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
+				      struct irq_domain *real_parent, struct msi_domain_info *info)
+{
+	struct irq_chip *chip = info->chip;
+
+	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
+		return false;
+
+	info->ops->msi_prepare = hv_msi_prepare;
+
+	chip->irq_set_affinity = irq_chip_set_affinity_parent;
+
+	if (IS_ENABLED(CONFIG_X86))
+		chip->flags |= IRQCHIP_MOVE_DEFERRED;
+
+	return true;
+}
+
+#define HV_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS		| \
+				    MSI_FLAG_USE_DEF_CHIP_OPS		| \
+				    MSI_FLAG_PCI_MSI_MASK_PARENT)
+#define HV_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI		| \
+				     MSI_FLAG_PCI_MSIX			| \
+				     MSI_GENERIC_FLAGS_MASK)
+
+static const struct msi_parent_ops hv_pcie_msi_parent_ops = {
+	.required_flags		= HV_PCIE_MSI_FLAGS_REQUIRED,
+	.supported_flags	= HV_PCIE_MSI_FLAGS_SUPPORTED,
+	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
+#ifdef CONFIG_X86
+	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
+#elif defined(CONFIG_ARM64)
+	.chip_flags		= MSI_CHIP_FLAG_SET_EOI,
+#endif
+	.prefix			= "HV-",
+	.init_dev_msi_info	= hv_pcie_init_dev_msi_info,
+};
+
 /* HW Interrupt Chip Descriptor */
 static struct irq_chip hv_msi_irq_chip = {
 	.name			= "Hyper-V PCIe MSI",
@@ -2108,7 +2144,6 @@ static struct irq_chip hv_msi_irq_chip = {
 	.irq_set_affinity	= irq_chip_set_affinity_parent,
 #ifdef CONFIG_X86
 	.irq_ack		= irq_chip_ack_parent,
-	.flags			= IRQCHIP_MOVE_DEFERRED,
 #elif defined(CONFIG_ARM64)
 	.irq_eoi		= irq_chip_eoi_parent,
 #endif
@@ -2116,9 +2151,37 @@ static struct irq_chip hv_msi_irq_chip = {
 	.irq_unmask		= hv_irq_unmask,
 };
 
-static struct msi_domain_ops hv_msi_ops = {
-	.msi_prepare	= hv_msi_prepare,
-	.msi_free	= hv_msi_free,
+static int hv_pcie_domain_alloc(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs,
+			       void *arg)
+{
+	/* TODO: move the content of hv_compose_msi_msg() in here */
+	int ret;
+
+	ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, arg);
+	if (ret < 0)
+		return ret;
+
+	for (int i = 0; i < nr_irqs; i++) {
+		irq_domain_set_info(d, virq + i, 0, &hv_msi_irq_chip, NULL, FLOW_HANDLER, NULL,
+				    FLOW_NAME);
+	}
+
+	return 0;
+}
+
+static void hv_pcie_domain_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs)
+{
+	struct msi_domain_info *info = d->host_data;
+
+	for (int i = 0; i < nr_irqs; i++)
+		hv_msi_free(d, info, virq + i);
+
+	irq_domain_free_irqs_top(d, virq, nr_irqs);
+}
+
+static const struct irq_domain_ops hv_pcie_domain_ops = {
+	.alloc	= hv_pcie_domain_alloc,
+	.free	= hv_pcie_domain_free,
 };
 
 /**
@@ -2136,17 +2199,14 @@ static struct msi_domain_ops hv_msi_ops = {
  */
 static int hv_pcie_init_irq_domain(struct hv_pcibus_device *hbus)
 {
-	hbus->msi_info.chip = &hv_msi_irq_chip;
-	hbus->msi_info.ops = &hv_msi_ops;
-	hbus->msi_info.flags = (MSI_FLAG_USE_DEF_DOM_OPS |
-		MSI_FLAG_USE_DEF_CHIP_OPS | MSI_FLAG_MULTI_PCI_MSI |
-		MSI_FLAG_PCI_MSIX);
-	hbus->msi_info.handler = FLOW_HANDLER;
-	hbus->msi_info.handler_name = FLOW_NAME;
-	hbus->msi_info.data = hbus;
-	hbus->irq_domain = pci_msi_create_irq_domain(hbus->fwnode,
-						     &hbus->msi_info,
-						     hv_pci_get_root_domain());
+	struct irq_domain_info info = {
+		.fwnode		= hbus->fwnode,
+		.ops		= &hv_pcie_domain_ops,
+		.host_data	= hbus,
+		.parent		= hv_pci_get_root_domain(),
+	};
+
+	hbus->irq_domain = msi_create_parent_irq_domain(&info, &hv_pcie_msi_parent_ops);
 	if (!hbus->irq_domain) {
 		dev_err(&hbus->hdev->device,
 			"Failed to build an MSI IRQ domain\n");
-- 
2.39.5

[PATCH 15/16] PCI: vmd: Convert to lock guards

[Nam Cao]

Convert lock/unlock pairs to lock guard and tidy up the code.

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Nirmal Patel <nirmal.patel@linux.intel.com>
Cc: Jonathan Derrick <jonathan.derrick@linux.dev>
---
 drivers/pci/controller/vmd.c | 73 ++++++++++++++----------------------
 1 file changed, 29 insertions(+), 44 deletions(-)

diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
index 375ce9d6d9f6a..d9b893bf4e456 100644
--- a/drivers/pci/controller/vmd.c
+++ b/drivers/pci/controller/vmd.c
@@ -180,13 +180,12 @@ static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 static void vmd_irq_enable(struct irq_data *data)
 {
 	struct vmd_irq *vmdirq = data->chip_data;
-	unsigned long flags;
 
-	raw_spin_lock_irqsave(&list_lock, flags);
-	WARN_ON(vmdirq->enabled);
-	list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list);
-	vmdirq->enabled = true;
-	raw_spin_unlock_irqrestore(&list_lock, flags);
+	scoped_guard(raw_spinlock_irqsave, &list_lock) {
+		WARN_ON(vmdirq->enabled);
+		list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list);
+		vmdirq->enabled = true;
+	}
 
 	data->chip->irq_unmask(data);
 }
@@ -194,16 +193,15 @@ static void vmd_irq_enable(struct irq_data *data)
 static void vmd_irq_disable(struct irq_data *data)
 {
 	struct vmd_irq *vmdirq = data->chip_data;
-	unsigned long flags;
 
 	data->chip->irq_mask(data);
 
-	raw_spin_lock_irqsave(&list_lock, flags);
-	if (vmdirq->enabled) {
-		list_del_rcu(&vmdirq->node);
-		vmdirq->enabled = false;
+	scoped_guard(raw_spinlock_irqsave, &list_lock) {
+		if (vmdirq->enabled) {
+			list_del_rcu(&vmdirq->node);
+			vmdirq->enabled = false;
+		}
 	}
-	raw_spin_unlock_irqrestore(&list_lock, flags);
 }
 
 static struct irq_chip vmd_msi_controller = {
@@ -225,7 +223,6 @@ static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info,
  */
 static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc)
 {
-	unsigned long flags;
 	int i, best;
 
 	if (vmd->msix_count == 1 + vmd->first_vec)
@@ -242,13 +239,13 @@ static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *d
 		return &vmd->irqs[vmd->first_vec];
 	}
 
-	raw_spin_lock_irqsave(&list_lock, flags);
-	best = vmd->first_vec + 1;
-	for (i = best; i < vmd->msix_count; i++)
-		if (vmd->irqs[i].count < vmd->irqs[best].count)
-			best = i;
-	vmd->irqs[best].count++;
-	raw_spin_unlock_irqrestore(&list_lock, flags);
+	scoped_guard(raw_spinlock_irq, &list_lock) {
+		best = vmd->first_vec + 1;
+		for (i = best; i < vmd->msix_count; i++)
+			if (vmd->irqs[i].count < vmd->irqs[best].count)
+				best = i;
+		vmd->irqs[best].count++;
+	}
 
 	return &vmd->irqs[best];
 }
@@ -277,14 +274,12 @@ static void vmd_msi_free(struct irq_domain *domain,
 			struct msi_domain_info *info, unsigned int virq)
 {
 	struct vmd_irq *vmdirq = irq_get_chip_data(virq);
-	unsigned long flags;
 
 	synchronize_srcu(&vmdirq->irq->srcu);
 
 	/* XXX: Potential optimization to rebalance */
-	raw_spin_lock_irqsave(&list_lock, flags);
-	vmdirq->irq->count--;
-	raw_spin_unlock_irqrestore(&list_lock, flags);
+	scoped_guard(raw_spinlock_irq, &list_lock)
+		vmdirq->irq->count--;
 
 	kfree(vmdirq);
 }
@@ -387,29 +382,24 @@ static int vmd_pci_read(struct pci_bus *bus, unsigned int devfn, int reg,
 {
 	struct vmd_dev *vmd = vmd_from_bus(bus);
 	void __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
-	unsigned long flags;
-	int ret = 0;
 
 	if (!addr)
 		return -EFAULT;
 
-	raw_spin_lock_irqsave(&vmd->cfg_lock, flags);
+	guard(raw_spinlock_irqsave)(&vmd->cfg_lock);
 	switch (len) {
 	case 1:
 		*value = readb(addr);
-		break;
+		return 0;
 	case 2:
 		*value = readw(addr);
-		break;
+		return 0;
 	case 4:
 		*value = readl(addr);
-		break;
+		return 0;
 	default:
-		ret = -EINVAL;
-		break;
+		return -EINVAL;
 	}
-	raw_spin_unlock_irqrestore(&vmd->cfg_lock, flags);
-	return ret;
 }
 
 /*
@@ -422,32 +412,27 @@ static int vmd_pci_write(struct pci_bus *bus, unsigned int devfn, int reg,
 {
 	struct vmd_dev *vmd = vmd_from_bus(bus);
 	void __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);
-	unsigned long flags;
-	int ret = 0;
 
 	if (!addr)
 		return -EFAULT;
 
-	raw_spin_lock_irqsave(&vmd->cfg_lock, flags);
+	guard(raw_spinlock_irqsave)(&vmd->cfg_lock);
 	switch (len) {
 	case 1:
 		writeb(value, addr);
 		readb(addr);
-		break;
+		return 0;
 	case 2:
 		writew(value, addr);
 		readw(addr);
-		break;
+		return 0;
 	case 4:
 		writel(value, addr);
 		readl(addr);
-		break;
+		return 0;
 	default:
-		ret = -EINVAL;
-		break;
+		return -EINVAL;
 	}
-	raw_spin_unlock_irqrestore(&vmd->cfg_lock, flags);
-	return ret;
 }
 
 static struct pci_ops vmd_ops = {
-- 
2.39.5

[PATCH 16/16] PCI: vmd: Switch to msi_create_parent_irq_domain()

[Nam Cao]

Move away from the legacy MSI domain setup, switch to use
msi_create_parent_irq_domain().

Signed-off-by: Nam Cao <namcao@linutronix.de>
---
Cc: Nirmal Patel <nirmal.patel@linux.intel.com>
Cc: Jonathan Derrick <jonathan.derrick@linux.dev>
---
 drivers/pci/controller/Kconfig |   1 +
 drivers/pci/controller/vmd.c   | 160 +++++++++++++++++----------------
 2 files changed, 82 insertions(+), 79 deletions(-)

diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index 8f56ffd029ba2..41748d083b933 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -156,6 +156,7 @@ config PCI_IXP4XX
 config VMD
 	depends on PCI_MSI && X86_64 && !UML
 	tristate "Intel Volume Management Device Driver"
+	select IRQ_MSI_LIB
 	help
 	  Adds support for the Intel Volume Management Device (VMD). VMD is a
 	  secondary PCI host bridge that allows PCI Express root ports,
diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
index d9b893bf4e456..38693a9487d9b 100644
--- a/drivers/pci/controller/vmd.c
+++ b/drivers/pci/controller/vmd.c
@@ -7,6 +7,7 @@
 #include <linux/device.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
+#include <linux/irqchip/irq-msi-lib.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/msi.h>
@@ -174,9 +175,6 @@ static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
 	msg->arch_addr_lo.destid_0_7 = index_from_irqs(vmd, irq);
 }
 
-/*
- * We rely on MSI_FLAG_USE_DEF_CHIP_OPS to set the IRQ mask/unmask ops.
- */
 static void vmd_irq_enable(struct irq_data *data)
 {
 	struct vmd_irq *vmdirq = data->chip_data;
@@ -186,7 +184,11 @@ static void vmd_irq_enable(struct irq_data *data)
 		list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list);
 		vmdirq->enabled = true;
 	}
+}
 
+static void vmd_pci_msi_enable(struct irq_data *data)
+{
+	vmd_irq_enable(data->parent_data);
 	data->chip->irq_unmask(data);
 }
 
@@ -194,8 +196,6 @@ static void vmd_irq_disable(struct irq_data *data)
 {
 	struct vmd_irq *vmdirq = data->chip_data;
 
-	data->chip->irq_mask(data);
-
 	scoped_guard(raw_spinlock_irqsave, &list_lock) {
 		if (vmdirq->enabled) {
 			list_del_rcu(&vmdirq->node);
@@ -204,19 +204,17 @@ static void vmd_irq_disable(struct irq_data *data)
 	}
 }
 
+static void vmd_pci_msi_disable(struct irq_data *data)
+{
+	data->chip->irq_mask(data);
+	vmd_irq_disable(data->parent_data);
+}
+
 static struct irq_chip vmd_msi_controller = {
 	.name			= "VMD-MSI",
-	.irq_enable		= vmd_irq_enable,
-	.irq_disable		= vmd_irq_disable,
 	.irq_compose_msi_msg	= vmd_compose_msi_msg,
 };
 
-static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info,
-				     msi_alloc_info_t *arg)
-{
-	return 0;
-}
-
 /*
  * XXX: We can be even smarter selecting the best IRQ once we solve the
  * affinity problem.
@@ -250,100 +248,110 @@ static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *d
 	return &vmd->irqs[best];
 }
 
-static int vmd_msi_init(struct irq_domain *domain, struct msi_domain_info *info,
-			unsigned int virq, irq_hw_number_t hwirq,
-			msi_alloc_info_t *arg)
+static void vmd_msi_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs);
+
+static int vmd_msi_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs,
+			 void *arg)
 {
-	struct msi_desc *desc = arg->desc;
-	struct vmd_dev *vmd = vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);
-	struct vmd_irq *vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL);
+	struct msi_desc *desc = ((msi_alloc_info_t *)arg)->desc;
+	struct vmd_dev *vmd = domain->host_data;
+	struct vmd_irq *vmdirq;
 
-	if (!vmdirq)
-		return -ENOMEM;
+	for (int i = 0; i < nr_irqs; ++i) {
+		vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL);
+		if (!vmdirq) {
+			vmd_msi_free(domain, virq, i);
+			return -ENOMEM;
+		}
 
-	INIT_LIST_HEAD(&vmdirq->node);
-	vmdirq->irq = vmd_next_irq(vmd, desc);
-	vmdirq->virq = virq;
+		INIT_LIST_HEAD(&vmdirq->node);
+		vmdirq->irq = vmd_next_irq(vmd, desc);
+		vmdirq->virq = virq + i;
+
+		irq_domain_set_info(domain, virq + i, vmdirq->irq->virq, &vmd_msi_controller,
+				    vmdirq, handle_untracked_irq, vmd, NULL);
+	}
 
-	irq_domain_set_info(domain, virq, vmdirq->irq->virq, info->chip, vmdirq,
-			    handle_untracked_irq, vmd, NULL);
 	return 0;
 }
 
-static void vmd_msi_free(struct irq_domain *domain,
-			struct msi_domain_info *info, unsigned int virq)
+static void vmd_msi_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs)
 {
 	struct vmd_irq *vmdirq = irq_get_chip_data(virq);
 
-	synchronize_srcu(&vmdirq->irq->srcu);
+	for (int i = 0; i < nr_irqs; ++i) {
+		synchronize_srcu(&vmdirq->irq->srcu);
 
-	/* XXX: Potential optimization to rebalance */
-	scoped_guard(raw_spinlock_irq, &list_lock)
-		vmdirq->irq->count--;
+		/* XXX: Potential optimization to rebalance */
+		scoped_guard(raw_spinlock_irq, &list_lock)
+			vmdirq->irq->count--;
 
-	kfree(vmdirq);
+		kfree(vmdirq);
+	}
 }
 
-static int vmd_msi_prepare(struct irq_domain *domain, struct device *dev,
-			   int nvec, msi_alloc_info_t *arg)
+static const struct irq_domain_ops vmd_msi_domain_ops = {
+	.alloc		= vmd_msi_alloc,
+	.free		= vmd_msi_free,
+};
+
+static bool vmd_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
+				  struct irq_domain *real_parent, struct msi_domain_info *info)
 {
-	struct pci_dev *pdev = to_pci_dev(dev);
-	struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
+	if (WARN_ON_ONCE(info->bus_token != DOMAIN_BUS_PCI_DEVICE_MSIX))
+		return false;
 
-	if (nvec > vmd->msix_count)
-		return vmd->msix_count;
+	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
+		return false;
 
-	memset(arg, 0, sizeof(*arg));
-	return 0;
+	info->chip->irq_enable		= vmd_pci_msi_enable;
+	info->chip->irq_disable		= vmd_pci_msi_disable;
+	return true;
 }
 
-static void vmd_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)
-{
-	arg->desc = desc;
-}
+#define VMD_MSI_FLAGS_SUPPORTED		(MSI_GENERIC_FLAGS_MASK | MSI_FLAG_PCI_MSIX)
+#define VMD_MSI_FLAGS_REQUIRED		(MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_NO_AFFINITY)
 
-static struct msi_domain_ops vmd_msi_domain_ops = {
-	.get_hwirq	= vmd_get_hwirq,
-	.msi_init	= vmd_msi_init,
-	.msi_free	= vmd_msi_free,
-	.msi_prepare	= vmd_msi_prepare,
-	.set_desc	= vmd_set_desc,
+static const struct msi_parent_ops vmd_msi_parent_ops = {
+	.supported_flags	= VMD_MSI_FLAGS_SUPPORTED,
+	.required_flags		= VMD_MSI_FLAGS_REQUIRED,
+	.bus_select_token	= DOMAIN_BUS_VMD_MSI,
+	.bus_select_mask	= MATCH_PCI_MSI,
+	.prefix			= "VMD-",
+	.init_dev_msi_info	= vmd_init_dev_msi_info,
 };
 
-static struct msi_domain_info vmd_msi_domain_info = {
-	.flags		= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
-			  MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX,
-	.ops		= &vmd_msi_domain_ops,
-	.chip		= &vmd_msi_controller,
-};
-
-static void vmd_set_msi_remapping(struct vmd_dev *vmd, bool enable)
-{
-	u16 reg;
-
-	pci_read_config_word(vmd->dev, PCI_REG_VMCONFIG, &reg);
-	reg = enable ? (reg & ~VMCONFIG_MSI_REMAP) :
-		       (reg | VMCONFIG_MSI_REMAP);
-	pci_write_config_word(vmd->dev, PCI_REG_VMCONFIG, reg);
-}
-
 static int vmd_create_irq_domain(struct vmd_dev *vmd)
 {
-	struct fwnode_handle *fn;
+	struct irq_domain_info info = {
+		.size		= vmd->msix_count,
+		.ops		= &vmd_msi_domain_ops,
+		.host_data	= vmd,
+	};
 
-	fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain);
-	if (!fn)
+	info.fwnode = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain);
+	if (!info.fwnode)
 		return -ENODEV;
 
-	vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info, NULL);
+	vmd->irq_domain = msi_create_parent_irq_domain(&info, &vmd_msi_parent_ops);
 	if (!vmd->irq_domain) {
-		irq_domain_free_fwnode(fn);
+		irq_domain_free_fwnode(info.fwnode);
 		return -ENODEV;
 	}
 
 	return 0;
 }
 
+static void vmd_set_msi_remapping(struct vmd_dev *vmd, bool enable)
+{
+	u16 reg;
+
+	pci_read_config_word(vmd->dev, PCI_REG_VMCONFIG, &reg);
+	reg = enable ? (reg & ~VMCONFIG_MSI_REMAP) :
+		       (reg | VMCONFIG_MSI_REMAP);
+	pci_write_config_word(vmd->dev, PCI_REG_VMCONFIG, reg);
+}
+
 static void vmd_remove_irq_domain(struct vmd_dev *vmd)
 {
 	/*
@@ -874,12 +882,6 @@ static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)
 		ret = vmd_create_irq_domain(vmd);
 		if (ret)
 			return ret;
-
-		/*
-		 * Override the IRQ domain bus token so the domain can be
-		 * distinguished from a regular PCI/MSI domain.
-		 */
-		irq_domain_update_bus_token(vmd->irq_domain, DOMAIN_BUS_VMD_MSI);
 	} else {
 		vmd_set_msi_remapping(vmd, false);
 	}
-- 
2.39.5

Re: [PATCH 06/16] PCI: iproc: Switch to msi_create_parent_irq_domain()

[Florian Fainelli]

On 6/26/25 07:47, Nam Cao wrote:
> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
> 
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Florian Fainelli <florian.fainelli@broadcom.com>
-- 
Florian

Re: [PATCH 05/16] PCI: brcmstb: Switch to msi_create_parent_irq_domain()

[Florian Fainelli]

On 6/26/25 07:47, Nam Cao wrote:
> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
> 
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Florian Fainelli <florian.fainelli@broadcom.com>
-- 
Florian

Re: [PATCH 01/16] PCI: dwc: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:47, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 02/16] PCI: mobiveil: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:47, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 03/16] PCI: aardvark: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:47, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 04/16] PCI: altera-msi: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:47, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 05/16] PCI: brcmstb: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:47, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 06/16] PCI: iproc: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:47, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 07/16] PCI: mediatek-gen3: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:47, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 08/16] PCI: mediatek: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:47, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 09/16] PCI: rcar-host: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:47, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 10/16] PCI: xilinx-xdma: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:48, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 11/16] PCI: xilinx-nwl: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:48, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 12/16] PCI: xilinx: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:48, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 13/16] PCI: plda: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:48, Nam Cao wrote:
> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:48, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 15/16] PCI: vmd: Convert to lock guards

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:48, Nam Cao wrote:
> Convert lock/unlock pairs to lock guard and tidy up the code.
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 16/16] PCI: vmd: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jun 26 2025 at 16:48, Nam Cao wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
>
> Signed-off-by: Nam Cao <namcao@linutronix.de>

Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

Re: [PATCH 00/16] PCI: MSI parent domain conversion

[Bjorn Helgaas]

On Thu, Jun 26, 2025 at 04:47:50PM +0200, Nam Cao wrote:
> The initial implementation of PCI/MSI interrupt domains in the hierarchical
> interrupt domain model used a shortcut by providing a global PCI/MSI
> domain.
> 
> This works because the PCI/MSI[X] hardware is standardized and uniform, but
> it violates the basic design principle of hierarchical interrupt domains:
> Each hardware block involved in the interrupt delivery chain should have a
> separate interrupt domain.
> 
> For PCI/MSI[X], the interrupt controller is per PCI device and not a global
> made-up entity.
> 
> Unsurprisingly, the shortcut turned out to have downsides as it does not
> allow dynamic allocation of interrupt vectors after initialization and it
> prevents supporting IMS on PCI. For further details, see:
> 
> https://lore.kernel.org/lkml/20221111120501.026511281@linutronix.de/
> 
> The solution is implementing per device MSI domains, this means the
> entities which provide global PCI/MSI domain so far have to implement MSI
> parent domain functionality instead.
> 
> This series converts the PCI controller drivers to implement MSI parent
> domain.
> 
>  drivers/pci/Kconfig                           |   1 +
>  drivers/pci/controller/Kconfig                |  11 +
>  drivers/pci/controller/dwc/Kconfig            |   1 +
>  .../pci/controller/dwc/pcie-designware-host.c |  68 ++----
>  drivers/pci/controller/dwc/pcie-designware.h  |   1 -
>  drivers/pci/controller/mobiveil/Kconfig       |   1 +
>  .../controller/mobiveil/pcie-mobiveil-host.c  |  42 ++--
>  .../pci/controller/mobiveil/pcie-mobiveil.h   |   1 -
>  drivers/pci/controller/pci-aardvark.c         |  59 ++---
>  drivers/pci/controller/pci-hyperv.c           |  98 ++++++--
>  drivers/pci/controller/pcie-altera-msi.c      |  44 ++--
>  drivers/pci/controller/pcie-brcmstb.c         |  44 ++--
>  drivers/pci/controller/pcie-iproc-msi.c       |  45 ++--
>  drivers/pci/controller/pcie-mediatek-gen3.c   |  67 ++---
>  drivers/pci/controller/pcie-mediatek.c        |  46 ++--
>  drivers/pci/controller/pcie-rcar-host.c       |  69 ++----
>  drivers/pci/controller/pcie-xilinx-dma-pl.c   |  48 ++--
>  drivers/pci/controller/pcie-xilinx-nwl.c      |  45 ++--
>  drivers/pci/controller/pcie-xilinx.c          |  55 +++--
>  drivers/pci/controller/plda/Kconfig           |   1 +
>  drivers/pci/controller/plda/pcie-plda-host.c  |  44 ++--
>  drivers/pci/controller/plda/pcie-plda.h       |   1 -
>  drivers/pci/controller/vmd.c                  | 229 +++++++++---------
>  23 files changed, 504 insertions(+), 517 deletions(-)

Looks good to me, thanks!  I think Mani will probably pick this up.

I might have included the specific "legacy MSI domain" thing you're
replacing.  It looks like you're replacing pci_msi_create_irq_domain()
with msi_create_parent_irq_domain()?

Minor merge conflict in pcie-mediatek-gen3.c with dcbea1c7e94e ("PCI:
mediatek-gen3: Use dev_fwnode() for irq_domain_create_linear()").  No
problem, we can easily fix that up.

The "++i" in vmd.c stuck out to me since "i++" is so much more common.

Acked-by: Bjorn Helgaas <bhelgaas@google.com>

RE: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Michael Kelley]

From: Nam Cao <namcao@linutronix.de> Sent: Thursday, June 26, 2025 7:48 AM
> 
> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().

From a build standpoint, this patch does not apply cleanly to
linux-next20250630. See also an issue below where a needed irq
function isn't exported.

At runtime, I've done basic smoke testing on an x86 VM in the Azure
cloud that has a Mellanox NIC VF and two NVMe devices as PCI devices.
So far everything looks good. But I'm still doing additional testing, and
I want to also test on an ARM64 VM. Please give me another day or two
to be completely satisfied.

Michael Kelley

> 
> Signed-off-by: Nam Cao <namcao@linutronix.de>
> ---
> Cc: K. Y. Srinivasan <kys@microsoft.com>
> Cc: Haiyang Zhang <haiyangz@microsoft.com>
> Cc: Wei Liu <wei.liu@kernel.org>
> Cc: Dexuan Cui <decui@microsoft.com>
> Cc: linux-hyperv@vger.kernel.org
> ---
>  drivers/pci/Kconfig                 |  1 +
>  drivers/pci/controller/pci-hyperv.c | 98 +++++++++++++++++++++++------
>  2 files changed, 80 insertions(+), 19 deletions(-)
> 
> diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
> index 9c0e4aaf4e8cb..9a249c65aedcd 100644
> --- a/drivers/pci/Kconfig
> +++ b/drivers/pci/Kconfig
> @@ -223,6 +223,7 @@ config PCI_HYPERV
>  	tristate "Hyper-V PCI Frontend"
>  	depends on ((X86 && X86_64) || ARM64) && HYPERV && PCI_MSI && SYSFS
>  	select PCI_HYPERV_INTERFACE
> +	select IRQ_MSI_LIB
>  	help
>  	  The PCI device frontend driver allows the kernel to import arbitrary
>  	  PCI devices from a PCI backend to support PCI driver domains.
> diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
> index ef5d655a0052c..3a24fadddb83b 100644
> --- a/drivers/pci/controller/pci-hyperv.c
> +++ b/drivers/pci/controller/pci-hyperv.c
> @@ -44,6 +44,7 @@
>  #include <linux/delay.h>
>  #include <linux/semaphore.h>
>  #include <linux/irq.h>
> +#include <linux/irqchip/irq-msi-lib.h>
>  #include <linux/msi.h>
>  #include <linux/hyperv.h>
>  #include <linux/refcount.h>
> @@ -508,7 +509,6 @@ struct hv_pcibus_device {
>  	struct list_head children;
>  	struct list_head dr_list;
> 
> -	struct msi_domain_info msi_info;
>  	struct irq_domain *irq_domain;
> 
>  	struct workqueue_struct *wq;
> @@ -1687,7 +1687,7 @@ static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
>  	struct msi_desc *msi = irq_data_get_msi_desc(irq_data);
> 
>  	pdev = msi_desc_to_pci_dev(msi);
> -	hbus = info->data;
> +	hbus = domain->host_data;
>  	int_desc = irq_data_get_irq_chip_data(irq_data);
>  	if (!int_desc)
>  		return;
> @@ -1705,7 +1705,6 @@ static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
> 
>  static void hv_irq_mask(struct irq_data *data)
>  {
> -	pci_msi_mask_irq(data);
>  	if (data->parent_data->chip->irq_mask)
>  		irq_chip_mask_parent(data);
>  }
> @@ -1716,7 +1715,6 @@ static void hv_irq_unmask(struct irq_data *data)
> 
>  	if (data->parent_data->chip->irq_unmask)
>  		irq_chip_unmask_parent(data);
> -	pci_msi_unmask_irq(data);
>  }
> 
>  struct compose_comp_ctxt {
> @@ -2101,6 +2099,44 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
>  	msg->data = 0;
>  }
> 
> +static bool hv_pcie_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
> +				      struct irq_domain *real_parent, struct msi_domain_info *info)
> +{
> +	struct irq_chip *chip = info->chip;
> +
> +	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
> +		return false;
> +
> +	info->ops->msi_prepare = hv_msi_prepare;
> +
> +	chip->irq_set_affinity = irq_chip_set_affinity_parent;
> +
> +	if (IS_ENABLED(CONFIG_X86))
> +		chip->flags |= IRQCHIP_MOVE_DEFERRED;
> +
> +	return true;
> +}
> +
> +#define HV_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS
> 	| \
> +				    MSI_FLAG_USE_DEF_CHIP_OPS		| \
> +				    MSI_FLAG_PCI_MSI_MASK_PARENT)
> +#define HV_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI
> 	| \
> +				     MSI_FLAG_PCI_MSIX			| \
> +				     MSI_GENERIC_FLAGS_MASK)
> +
> +static const struct msi_parent_ops hv_pcie_msi_parent_ops = {
> +	.required_flags		= HV_PCIE_MSI_FLAGS_REQUIRED,
> +	.supported_flags	= HV_PCIE_MSI_FLAGS_SUPPORTED,
> +	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
> +#ifdef CONFIG_X86
> +	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
> +#elif defined(CONFIG_ARM64)
> +	.chip_flags		= MSI_CHIP_FLAG_SET_EOI,
> +#endif
> +	.prefix			= "HV-",
> +	.init_dev_msi_info	= hv_pcie_init_dev_msi_info,
> +};
> +
>  /* HW Interrupt Chip Descriptor */
>  static struct irq_chip hv_msi_irq_chip = {
>  	.name			= "Hyper-V PCIe MSI",
> @@ -2108,7 +2144,6 @@ static struct irq_chip hv_msi_irq_chip = {
>  	.irq_set_affinity	= irq_chip_set_affinity_parent,
>  #ifdef CONFIG_X86
>  	.irq_ack		= irq_chip_ack_parent,
> -	.flags			= IRQCHIP_MOVE_DEFERRED,
>  #elif defined(CONFIG_ARM64)
>  	.irq_eoi		= irq_chip_eoi_parent,
>  #endif
> @@ -2116,9 +2151,37 @@ static struct irq_chip hv_msi_irq_chip = {
>  	.irq_unmask		= hv_irq_unmask,
>  };
> 
> -static struct msi_domain_ops hv_msi_ops = {
> -	.msi_prepare	= hv_msi_prepare,
> -	.msi_free	= hv_msi_free,
> +static int hv_pcie_domain_alloc(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs,
> +			       void *arg)
> +{
> +	/* TODO: move the content of hv_compose_msi_msg() in here */
> +	int ret;
> +
> +	ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, arg);
> +	if (ret < 0)
> +		return ret;
> +
> +	for (int i = 0; i < nr_irqs; i++) {
> +		irq_domain_set_info(d, virq + i, 0, &hv_msi_irq_chip, NULL, FLOW_HANDLER, NULL,
> +				    FLOW_NAME);
> +	}
> +
> +	return 0;
> +}
> +
> +static void hv_pcie_domain_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs)
> +{
> +	struct msi_domain_info *info = d->host_data;
> +
> +	for (int i = 0; i < nr_irqs; i++)
> +		hv_msi_free(d, info, virq + i);
> +
> +	irq_domain_free_irqs_top(d, virq, nr_irqs);

This code can be built as a module, so irq_domain_free_irqs_top() needs to be
exported, which it currently is not.

> +}
> +
> +static const struct irq_domain_ops hv_pcie_domain_ops = {
> +	.alloc	= hv_pcie_domain_alloc,
> +	.free	= hv_pcie_domain_free,
>  };
> 
>  /**
> @@ -2136,17 +2199,14 @@ static struct msi_domain_ops hv_msi_ops = {
>   */
>  static int hv_pcie_init_irq_domain(struct hv_pcibus_device *hbus)
>  {
> -	hbus->msi_info.chip = &hv_msi_irq_chip;
> -	hbus->msi_info.ops = &hv_msi_ops;
> -	hbus->msi_info.flags = (MSI_FLAG_USE_DEF_DOM_OPS |
> -		MSI_FLAG_USE_DEF_CHIP_OPS | MSI_FLAG_MULTI_PCI_MSI |
> -		MSI_FLAG_PCI_MSIX);
> -	hbus->msi_info.handler = FLOW_HANDLER;
> -	hbus->msi_info.handler_name = FLOW_NAME;
> -	hbus->msi_info.data = hbus;
> -	hbus->irq_domain = pci_msi_create_irq_domain(hbus->fwnode,
> -						     &hbus->msi_info,
> -						     hv_pci_get_root_domain());
> +	struct irq_domain_info info = {
> +		.fwnode		= hbus->fwnode,
> +		.ops		= &hv_pcie_domain_ops,
> +		.host_data	= hbus,
> +		.parent		= hv_pci_get_root_domain(),
> +	};
> +
> +	hbus->irq_domain = msi_create_parent_irq_domain(&info, &hv_pcie_msi_parent_ops);
>  	if (!hbus->irq_domain) {
>  		dev_err(&hbus->hdev->device,
>  			"Failed to build an MSI IRQ domain\n");
> --
> 2.39.5
> 

RE: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jul 03 2025 at 17:41, Michael Kelley wrote:
> From: Nam Cao <namcao@linutronix.de> Sent: Thursday, June 26, 2025 7:48 AM
>> 
>> Move away from the legacy MSI domain setup, switch to use
>> msi_create_parent_irq_domain().
>
> From a build standpoint, this patch does not apply cleanly to
> linux-next20250630. See also an issue below where a needed irq
> function isn't exported.

Does it conflict against the PCI tree?

> At runtime, I've done basic smoke testing on an x86 VM in the Azure
> cloud that has a Mellanox NIC VF and two NVMe devices as PCI devices.
> So far everything looks good. But I'm still doing additional testing, and
> I want to also test on an ARM64 VM. Please give me another day or two
> to be completely satisfied.

Sure.
>> +static void hv_pcie_domain_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs)
>> +{
>> +	struct msi_domain_info *info = d->host_data;
>> +
>> +	for (int i = 0; i < nr_irqs; i++)
>> +		hv_msi_free(d, info, virq + i);
>> +
>> +	irq_domain_free_irqs_top(d, virq, nr_irqs);
>
> This code can be built as a module, so irq_domain_free_irqs_top() needs to be
> exported, which it currently is not.

Nam, can you please create a seperate patch, which exports this and take
care of the conflict?

Thanks,

        tglx

RE: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Michael Kelley]

From: Thomas Gleixner <tglx@linutronix.de> Sent: Thursday, July 3, 2025 1:00 PM
> 
> On Thu, Jul 03 2025 at 17:41, Michael Kelley wrote:
> > From: Nam Cao <namcao@linutronix.de> Sent: Thursday, June 26, 2025 7:48 AM
> >>
> >> Move away from the legacy MSI domain setup, switch to use
> >> msi_create_parent_irq_domain().
> >
> > From a build standpoint, this patch does not apply cleanly to
> > linux-next20250630. See also an issue below where a needed irq
> > function isn't exported.
> 
> Does it conflict against the PCI tree?

There's no conflict in the "next" or "for-linus" tags in
https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git/.

The conflict is with Patch 2 of this series:

https://lore.kernel.org/linux-hyperv/1749650984-9193-1-git-send-email-shradhagupta@linux.microsoft.com/

which is in netdev/net-next.

Michael

> 
> > At runtime, I've done basic smoke testing on an x86 VM in the Azure
> > cloud that has a Mellanox NIC VF and two NVMe devices as PCI devices.
> > So far everything looks good. But I'm still doing additional testing, and
> > I want to also test on an ARM64 VM. Please give me another day or two
> > to be completely satisfied.
> 
> Sure.
> >> +static void hv_pcie_domain_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs)
> >> +{
> >> +	struct msi_domain_info *info = d->host_data;
> >> +
> >> +	for (int i = 0; i < nr_irqs; i++)
> >> +		hv_msi_free(d, info, virq + i);
> >> +
> >> +	irq_domain_free_irqs_top(d, virq, nr_irqs);
> >
> > This code can be built as a module, so irq_domain_free_irqs_top() needs to be
> > exported, which it currently is not.
> 
> Nam, can you please create a seperate patch, which exports this and take
> care of the conflict?
> 
> Thanks,
> 
>         tglx

Re: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Nam Cao]

On Thu, Jul 03, 2025 at 08:15:07PM +0000, Michael Kelley wrote:
> From: Thomas Gleixner <tglx@linutronix.de> Sent: Thursday, July 3, 2025 1:00 PM
> > 
> > On Thu, Jul 03 2025 at 17:41, Michael Kelley wrote:
> > > From: Nam Cao <namcao@linutronix.de> Sent: Thursday, June 26, 2025 7:48 AM
> > >>
> > >> Move away from the legacy MSI domain setup, switch to use
> > >> msi_create_parent_irq_domain().
> > >
> > > From a build standpoint, this patch does not apply cleanly to
> > > linux-next20250630. See also an issue below where a needed irq
> > > function isn't exported.
> > 
> > Does it conflict against the PCI tree?
> 
> There's no conflict in the "next" or "for-linus" tags in
> https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git/.
> 
> The conflict is with Patch 2 of this series:
> 
> https://lore.kernel.org/linux-hyperv/1749650984-9193-1-git-send-email-shradhagupta@linux.microsoft.com/
> 
> which is in netdev/net-next.

I need some guidance here. If I make it apply cleanly to linux-next, it
won't apply to pci tree.

I saw this type of conflict being resolved during merging to Linus's tree.
Shouldn't we do the same for this case?

> Michael
> 
> > 
> > > At runtime, I've done basic smoke testing on an x86 VM in the Azure
> > > cloud that has a Mellanox NIC VF and two NVMe devices as PCI devices.
> > > So far everything looks good. But I'm still doing additional testing, and
> > > I want to also test on an ARM64 VM. Please give me another day or two
> > > to be completely satisfied.

Good to hear, thanks!

> > Sure.
> > >> +static void hv_pcie_domain_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs)
> > >> +{
> > >> +	struct msi_domain_info *info = d->host_data;
> > >> +
> > >> +	for (int i = 0; i < nr_irqs; i++)
> > >> +		hv_msi_free(d, info, virq + i);
> > >> +
> > >> +	irq_domain_free_irqs_top(d, virq, nr_irqs);
> > >
> > > This code can be built as a module, so irq_domain_free_irqs_top() needs to be
> > > exported, which it currently is not.
> > 
> > Nam, can you please create a seperate patch, which exports this and take
> > care of the conflict?

Will do.

Best regards,
Nam

RE: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jul 03 2025 at 20:15, Michael Kelley wrote:
> From: Thomas Gleixner <tglx@linutronix.de> Sent: Thursday, July 3, 2025 1:00 PM
>> Does it conflict against the PCI tree?
>
> There's no conflict in the "next" or "for-linus" tags in
> https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git/.
>
> The conflict is with Patch 2 of this series:
>
> https://lore.kernel.org/linux-hyperv/1749650984-9193-1-git-send-email-shradhagupta@linux.microsoft.com/
>
> which is in netdev/net-next.

That's a trivial one. There are two ways to handle it:

  1) Take it through the PCI tree and provide a conflict resolution for
     linux-next and later for Linus as reference.

  2) Route it through the net-next tree with an updated patch.

As there are no further dependencies (aside of the missing export which
is needed anyway) it's obvious to pick #2 as it creates the least
headaches. Assumed that the PCI folks have no objections.

Michael, as you have resolved the conflict already, can you please
either take care of it yourself or provide the resolution here as
reference for Nam?

Thanks,

        tglx

Re: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Thomas Gleixner]

On Thu, Jul 03 2025 at 23:00, Nam Cao wrote:
> On Thu, Jul 03, 2025 at 08:15:07PM +0000, Michael Kelley wrote:
>> which is in netdev/net-next.
>
> I need some guidance here. If I make it apply cleanly to linux-next, it
> won't apply to pci tree.
>
> I saw this type of conflict being resolved during merging to Linus's tree.
> Shouldn't we do the same for this case?

There are many ways to skin a cat. See my other reply.

RE: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Michael Kelley]

From: Thomas Gleixner <tglx@linutronix.de> Sent: Thursday, July 3, 2025 2:22 PM
> 
> On Thu, Jul 03 2025 at 20:15, Michael Kelley wrote:
> > From: Thomas Gleixner <tglx@linutronix.de> Sent: Thursday, July 3, 2025 1:00 PM
> >> Does it conflict against the PCI tree?
> >
> > There's no conflict in the "next" or "for-linus" tags in
> > https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git/
> >
> > The conflict is with Patch 2 of this series:
> >
> > https://lore.kernel.org/linux-hyperv/1749650984-9193-1-git-send-email-shradhagupta@linux.microsoft.com/
> >
> > which is in netdev/net-next.
> 
> That's a trivial one. There are two ways to handle it:
> 
>   1) Take it through the PCI tree and provide a conflict resolution for
>      linux-next and later for Linus as reference.
> 
>   2) Route it through the net-next tree with an updated patch.
> 
> As there are no further dependencies (aside of the missing export which
> is needed anyway) it's obvious to pick #2 as it creates the least
> headaches. Assumed that the PCI folks have no objections.
> 
> Michael, as you have resolved the conflict already, can you please
> either take care of it yourself or provide the resolution here as
> reference for Nam?

I haven't resolved the conflict. As a shortcut for testing I just
removed the conflicting patch since it is for a Microsoft custom NIC
("MANA") that's not in the configuration I'm testing with. I'll have to
look more closely to figure out the resolution.

Separately, this patch (the switch to misc_create_parent_irq_domain)
isn't working for Linux VMs on Hyper-V on ARM64. The initial symptom
is that interrupts from the NVMe controller aren't getting handled
and everything hangs. Here's the dmesg output:

[   84.463419] hv_vmbus: registering driver hv_pci
[   84.463875] hv_pci abee639e-0b9d-49b7-9a07-c54ba8cd5734: PCI VMBus probing: Using version 0x10004
[   84.464518] hv_pci abee639e-0b9d-49b7-9a07-c54ba8cd5734: PCI host bridge to bus 0b9d:00
[   84.464529] pci_bus 0b9d:00: root bus resource [mem 0xfc0000000-0xfc00fffff window]
[   84.464531] pci_bus 0b9d:00: No busn resource found for root bus, will use [bus 00-ff]
[   84.465211] pci 0b9d:00:00.0: [1414:b111] type 00 class 0x010802 PCIe Endpoint
[   84.466657] pci 0b9d:00:00.0: BAR 0 [mem 0xfc0000000-0xfc00fffff 64bit]
[   84.481923] pci_bus 0b9d:00: busn_res: [bus 00-ff] end is updated to 00
[   84.481936] pci 0b9d:00:00.0: BAR 0 [mem 0xfc0000000-0xfc00fffff 64bit]: assigned
[   84.482413] nvme nvme0: pci function 0b9d:00:00.0
[   84.482513] nvme 0b9d:00:00.0: enabling device (0000 -> 0002)
[   84.556871] irq 17, desc: 00000000e8529819, depth: 0, count: 0, unhandled: 0
[   84.556883] ->handle_irq():  0000000062fa78bc, handle_bad_irq+0x0/0x270
[   84.556892] ->irq_data.chip(): 00000000ba07832f, 0xffff00011469dc30
[   84.556895] ->action(): 0000000069f160b3
[   84.556896] ->action->handler(): 00000000e15d8191, nvme_irq+0x0/0x3e8
[  172.307920] watchdog: BUG: soft lockup - CPU#6 stuck for 26s! [kworker/6:1H:195]

Everything looks normal up to the "irq 17" line.  Meanwhile, the device probe path
is waiting with this stack trace, which I suspect is the first Interaction with the NVMe
controller:

[<0>] blk_execute_rq+0x1ec/0x348
[<0>] nvme_execute_rq+0x20/0x68
[<0>] __nvme_submit_sync_cmd+0xc8/0x170
[<0>] nvme_identify_ctrl.isra.0+0x90/0xf0
[<0>] nvme_init_identify+0x44/0xee0
[<0>] nvme_init_ctrl_finish+0x84/0x370
[<0>] nvme_probe+0x668/0x7d8
[<0>] local_pci_probe+0x48/0xd0
[<0>] pci_device_probe+0xd0/0x248
[<0>] really_probe+0xd4/0x388
[<0>] __driver_probe_device+0x90/0x1a8
[<0>] driver_probe_device+0x48/0x150
[<0>] __device_attach_driver+0xe0/0x1b8
[<0>] bus_for_each_drv+0x8c/0xf8
[<0>] __device_attach+0x104/0x1e8
[<0>] device_attach+0x1c/0x30
[<0>] pci_bus_add_device+0xe0/0x188
[<0>] pci_bus_add_devices+0x40/0x98
[<0>] hv_pci_probe+0x4b0/0x690 [pci_hyperv]
[<0>] vmbus_probe+0x4c/0xb0 [hv_vmbus]
[<0>] really_probe+0xd4/0x388
[<0>] __driver_probe_device+0x90/0x1a8
[<0>] driver_probe_device+0x48/0x150
[<0>] __driver_attach+0xe8/0x1c8
[<0>] bus_for_each_dev+0x80/0xf0
[<0>] driver_attach+0x2c/0x40
[<0>] bus_add_driver+0x118/0x260
[<0>] driver_register+0x68/0x138
[<0>] __vmbus_driver_register+0x70/0x98 [hv_vmbus]
[<0>] init_hv_pci_drv+0x1b8/0xfff8 [pci_hyperv]
[<0>] do_one_initcall+0x4c/0x2c8
[<0>] do_init_module+0x60/0x280
[<0>] load_module+0x2318/0x2448
[<0>] init_module_from_file+0x94/0xe0
[<0>] __arm64_sys_finit_module+0x228/0x3e8
[<0>] invoke_syscall+0x6c/0xf8
[<0>] el0_svc_common.constprop.0+0xc8/0xf0
[<0>] do_el0_svc+0x24/0x38
[<0>] el0_svc+0x40/0x1a0
[<0>] el0t_64_sync_handler+0xd0/0xe8
[<0>] el0t_64_sync+0x1b0/0x1b8

I'll try to figure out what's going wrong.

Michael

Re: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Nam Cao]

On Fri, Jul 04, 2025 at 02:27:01AM +0000, Michael Kelley wrote:
> I haven't resolved the conflict. As a shortcut for testing I just
> removed the conflicting patch since it is for a Microsoft custom NIC
> ("MANA") that's not in the configuration I'm testing with. I'll have to
> look more closely to figure out the resolution.
> 
> Separately, this patch (the switch to misc_create_parent_irq_domain)
> isn't working for Linux VMs on Hyper-V on ARM64. The initial symptom
> is that interrupts from the NVMe controller aren't getting handled
> and everything hangs. Here's the dmesg output:
> 
> [   84.463419] hv_vmbus: registering driver hv_pci
> [   84.463875] hv_pci abee639e-0b9d-49b7-9a07-c54ba8cd5734: PCI VMBus probing: Using version 0x10004
> [   84.464518] hv_pci abee639e-0b9d-49b7-9a07-c54ba8cd5734: PCI host bridge to bus 0b9d:00
> [   84.464529] pci_bus 0b9d:00: root bus resource [mem 0xfc0000000-0xfc00fffff window]
> [   84.464531] pci_bus 0b9d:00: No busn resource found for root bus, will use [bus 00-ff]
> [   84.465211] pci 0b9d:00:00.0: [1414:b111] type 00 class 0x010802 PCIe Endpoint
> [   84.466657] pci 0b9d:00:00.0: BAR 0 [mem 0xfc0000000-0xfc00fffff 64bit]
> [   84.481923] pci_bus 0b9d:00: busn_res: [bus 00-ff] end is updated to 00
> [   84.481936] pci 0b9d:00:00.0: BAR 0 [mem 0xfc0000000-0xfc00fffff 64bit]: assigned
> [   84.482413] nvme nvme0: pci function 0b9d:00:00.0
> [   84.482513] nvme 0b9d:00:00.0: enabling device (0000 -> 0002)
> [   84.556871] irq 17, desc: 00000000e8529819, depth: 0, count: 0, unhandled: 0
> [   84.556883] ->handle_irq():  0000000062fa78bc, handle_bad_irq+0x0/0x270
> [   84.556892] ->irq_data.chip(): 00000000ba07832f, 0xffff00011469dc30
> [   84.556895] ->action(): 0000000069f160b3
> [   84.556896] ->action->handler(): 00000000e15d8191, nvme_irq+0x0/0x3e8
> [  172.307920] watchdog: BUG: soft lockup - CPU#6 stuck for 26s! [kworker/6:1H:195]

Thanks for the report.

On arm64, this driver relies on the parent irq domain to set handler. So
the driver must not overwrite it to NULL.

This should cures it:

diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
index 3a24fadddb83..f4a435b0456c 100644
--- a/drivers/pci/controller/pci-hyperv.c
+++ b/drivers/pci/controller/pci-hyperv.c
@@ -577,8 +577,6 @@ static void hv_pci_onchannelcallback(void *context);
 
 #ifdef CONFIG_X86
 #define DELIVERY_MODE	APIC_DELIVERY_MODE_FIXED
-#define FLOW_HANDLER	handle_edge_irq
-#define FLOW_NAME	"edge"
 
 static int hv_pci_irqchip_init(void)
 {
@@ -723,8 +721,6 @@ static void hv_arch_irq_unmask(struct irq_data *data)
 #define HV_PCI_MSI_SPI_START	64
 #define HV_PCI_MSI_SPI_NR	(1020 - HV_PCI_MSI_SPI_START)
 #define DELIVERY_MODE		0
-#define FLOW_HANDLER		NULL
-#define FLOW_NAME		NULL
 #define hv_msi_prepare		NULL
 
 struct hv_pci_chip_data {
@@ -2162,8 +2158,9 @@ static int hv_pcie_domain_alloc(struct irq_domain *d, unsigned int virq, unsigne
 		return ret;
 
 	for (int i = 0; i < nr_irqs; i++) {
-		irq_domain_set_info(d, virq + i, 0, &hv_msi_irq_chip, NULL, FLOW_HANDLER, NULL,
-				    FLOW_NAME);
+		irq_domain_set_hwirq_and_chip(d, virq + i, 0, &hv_msi_irq_chip, NULL);
+		if (IS_ENABLED(CONFIG_X86))
+			__irq_set_handler(virq + i, handle_edge_irq, 0, "edge");
 	}
 
 	return 0;

Re: [PATCH 00/16] PCI: MSI parent domain conversion

[Nam Cao]

On Thu, Jul 03, 2025 at 12:28:01PM -0500, Bjorn Helgaas wrote:
> On Thu, Jun 26, 2025 at 04:47:50PM +0200, Nam Cao wrote:
> > The initial implementation of PCI/MSI interrupt domains in the hierarchical
> > interrupt domain model used a shortcut by providing a global PCI/MSI
> > domain.
> > 
> > This works because the PCI/MSI[X] hardware is standardized and uniform, but
> > it violates the basic design principle of hierarchical interrupt domains:
> > Each hardware block involved in the interrupt delivery chain should have a
> > separate interrupt domain.
> > 
> > For PCI/MSI[X], the interrupt controller is per PCI device and not a global
> > made-up entity.
> > 
> > Unsurprisingly, the shortcut turned out to have downsides as it does not
> > allow dynamic allocation of interrupt vectors after initialization and it
> > prevents supporting IMS on PCI. For further details, see:
> > 
> > https://lore.kernel.org/lkml/20221111120501.026511281@linutronix.de/
> > 
> > The solution is implementing per device MSI domains, this means the
> > entities which provide global PCI/MSI domain so far have to implement MSI
> > parent domain functionality instead.
> > 
> > This series converts the PCI controller drivers to implement MSI parent
> > domain.
> > 
> >  drivers/pci/Kconfig                           |   1 +
> >  drivers/pci/controller/Kconfig                |  11 +
> >  drivers/pci/controller/dwc/Kconfig            |   1 +
> >  .../pci/controller/dwc/pcie-designware-host.c |  68 ++----
> >  drivers/pci/controller/dwc/pcie-designware.h  |   1 -
> >  drivers/pci/controller/mobiveil/Kconfig       |   1 +
> >  .../controller/mobiveil/pcie-mobiveil-host.c  |  42 ++--
> >  .../pci/controller/mobiveil/pcie-mobiveil.h   |   1 -
> >  drivers/pci/controller/pci-aardvark.c         |  59 ++---
> >  drivers/pci/controller/pci-hyperv.c           |  98 ++++++--
> >  drivers/pci/controller/pcie-altera-msi.c      |  44 ++--
> >  drivers/pci/controller/pcie-brcmstb.c         |  44 ++--
> >  drivers/pci/controller/pcie-iproc-msi.c       |  45 ++--
> >  drivers/pci/controller/pcie-mediatek-gen3.c   |  67 ++---
> >  drivers/pci/controller/pcie-mediatek.c        |  46 ++--
> >  drivers/pci/controller/pcie-rcar-host.c       |  69 ++----
> >  drivers/pci/controller/pcie-xilinx-dma-pl.c   |  48 ++--
> >  drivers/pci/controller/pcie-xilinx-nwl.c      |  45 ++--
> >  drivers/pci/controller/pcie-xilinx.c          |  55 +++--
> >  drivers/pci/controller/plda/Kconfig           |   1 +
> >  drivers/pci/controller/plda/pcie-plda-host.c  |  44 ++--
> >  drivers/pci/controller/plda/pcie-plda.h       |   1 -
> >  drivers/pci/controller/vmd.c                  | 229 +++++++++---------
> >  23 files changed, 504 insertions(+), 517 deletions(-)
> 
> Looks good to me, thanks!  I think Mani will probably pick this up.
> 
> I might have included the specific "legacy MSI domain" thing you're
> replacing.  It looks like you're replacing pci_msi_create_irq_domain()
> with msi_create_parent_irq_domain()?

Yes, pci_msi_create_irq_domain() is legacy. We will delete it once
everything is converted.

> Minor merge conflict in pcie-mediatek-gen3.c with dcbea1c7e94e ("PCI:
> mediatek-gen3: Use dev_fwnode() for irq_domain_create_linear()").  No
> problem, we can easily fix that up.

Thanks!

> The "++i" in vmd.c stuck out to me since "i++" is so much more common.

I always do "++i", maybe I'm the weird one..

Best regards,
Nam

RE: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Michael Kelley]

From: Nam Cao <namcao@linutronix.de> Sent: Thursday, July 3, 2025 9:33 PM
> 
> On Fri, Jul 04, 2025 at 02:27:01AM +0000, Michael Kelley wrote:
> > I haven't resolved the conflict. As a shortcut for testing I just
> > removed the conflicting patch since it is for a Microsoft custom NIC
> > ("MANA") that's not in the configuration I'm testing with. I'll have to
> > look more closely to figure out the resolution.
> >
> > Separately, this patch (the switch to misc_create_parent_irq_domain)
> > isn't working for Linux VMs on Hyper-V on ARM64. The initial symptom
> > is that interrupts from the NVMe controller aren't getting handled
> > and everything hangs. Here's the dmesg output:
> >
> > [   84.463419] hv_vmbus: registering driver hv_pci
> > [   84.463875] hv_pci abee639e-0b9d-49b7-9a07-c54ba8cd5734: PCI VMBus probing: Using version 0x10004
> > [   84.464518] hv_pci abee639e-0b9d-49b7-9a07-c54ba8cd5734: PCI host bridge to bus 0b9d:00
> > [   84.464529] pci_bus 0b9d:00: root bus resource [mem 0xfc0000000-0xfc00fffff window]
> > [   84.464531] pci_bus 0b9d:00: No busn resource found for root bus, will use [bus 00-ff]
> > [   84.465211] pci 0b9d:00:00.0: [1414:b111] type 00 class 0x010802 PCIe Endpoint
> > [   84.466657] pci 0b9d:00:00.0: BAR 0 [mem 0xfc0000000-0xfc00fffff 64bit]
> > [   84.481923] pci_bus 0b9d:00: busn_res: [bus 00-ff] end is updated to 00
> > [   84.481936] pci 0b9d:00:00.0: BAR 0 [mem 0xfc0000000-0xfc00fffff 64bit]: assigned
> > [   84.482413] nvme nvme0: pci function 0b9d:00:00.0
> > [   84.482513] nvme 0b9d:00:00.0: enabling device (0000 -> 0002)
> > [   84.556871] irq 17, desc: 00000000e8529819, depth: 0, count: 0, unhandled: 0
> > [   84.556883] ->handle_irq():  0000000062fa78bc, handle_bad_irq+0x0/0x270
> > [   84.556892] ->irq_data.chip(): 00000000ba07832f, 0xffff00011469dc30
> > [   84.556895] ->action(): 0000000069f160b3
> > [   84.556896] ->action->handler(): 00000000e15d8191, nvme_irq+0x0/0x3e8
> > [  172.307920] watchdog: BUG: soft lockup - CPU#6 stuck for 26s! [kworker/6:1H:195]
> 
> Thanks for the report.
> 
> On arm64, this driver relies on the parent irq domain to set handler. So
> the driver must not overwrite it to NULL.
> 
> This should cures it:
> 
> diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
> index 3a24fadddb83..f4a435b0456c 100644
> --- a/drivers/pci/controller/pci-hyperv.c
> +++ b/drivers/pci/controller/pci-hyperv.c
> @@ -577,8 +577,6 @@ static void hv_pci_onchannelcallback(void *context);
> 
>  #ifdef CONFIG_X86
>  #define DELIVERY_MODE	APIC_DELIVERY_MODE_FIXED
> -#define FLOW_HANDLER	handle_edge_irq
> -#define FLOW_NAME	"edge"
> 
>  static int hv_pci_irqchip_init(void)
>  {
> @@ -723,8 +721,6 @@ static void hv_arch_irq_unmask(struct irq_data *data)
>  #define HV_PCI_MSI_SPI_START	64
>  #define HV_PCI_MSI_SPI_NR	(1020 - HV_PCI_MSI_SPI_START)
>  #define DELIVERY_MODE		0
> -#define FLOW_HANDLER		NULL
> -#define FLOW_NAME		NULL
>  #define hv_msi_prepare		NULL
> 
>  struct hv_pci_chip_data {
> @@ -2162,8 +2158,9 @@ static int hv_pcie_domain_alloc(struct irq_domain *d,
> unsigned int virq, unsigne
>  		return ret;
> 
>  	for (int i = 0; i < nr_irqs; i++) {
> -		irq_domain_set_info(d, virq + i, 0, &hv_msi_irq_chip, NULL, FLOW_HANDLER, NULL,
> -				    FLOW_NAME);
> +		irq_domain_set_hwirq_and_chip(d, virq + i, 0, &hv_msi_irq_chip, NULL);
> +		if (IS_ENABLED(CONFIG_X86))
> +			__irq_set_handler(virq + i, handle_edge_irq, 0, "edge");
>  	}
> 
>  	return 0;

Yes, that fixes the problem. Linux now boots with the PCI NIC VF and two
NVMe controllers being visible and operational. Thanks for the fix! It
would have taken me a while to figure it out.

I want to do some additional testing tomorrow, and look more closely at the
code, but now I have something that works well enough to make further
progress.

Michael

Re: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[kernel test robot]

Hi Nam,

kernel test robot noticed the following build errors:

[auto build test ERROR on pci/next]
[cannot apply to pci/for-linus linus/master v6.16-rc4 next-20250704]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Nam-Cao/PCI-dwc-Switch-to-msi_create_parent_irq_domain/20250626-230029
base:   https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git next
patch link:    https://lore.kernel.org/r/024f0122314198fe0a42fef01af53e8953a687ec.1750858083.git.namcao%40linutronix.de
patch subject: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()
config: x86_64-rhel-9.4 (https://download.01.org/0day-ci/archive/20250705/202507051005.SB1ajh7M-lkp@intel.com/config)
compiler: gcc-12 (Debian 12.2.0-14+deb12u1) 12.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250705/202507051005.SB1ajh7M-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202507051005.SB1ajh7M-lkp@intel.com/

All errors (new ones prefixed by >>, old ones prefixed by <<):

>> ERROR: modpost: "irq_domain_free_irqs_top" [drivers/pci/controller/pci-hyperv.ko] undefined!

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

RE: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Michael Kelley]

From: Nam Cao <namcao@linutronix.de> Sent: Thursday, June 26, 2025 7:48 AM
> 
> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().

With the additional tweak to this patch that you supplied separately,
everything in my testing on both x86 and arm64 seems to work OK. So
that's all good.

On arm64, I did notice the following IRQ domain information from
/sys/kernel/debug/irq/domains:

# cat HV-PCI-MSIX-1e03\:00\:00.0-12
name:   HV-PCI-MSIX-1e03:00:00.0-12
 size:   0
 mapped: 7
 flags:  0x00000213
            IRQ_DOMAIN_FLAG_HIERARCHY
            IRQ_DOMAIN_NAME_ALLOCATED
            IRQ_DOMAIN_FLAG_MSI
            IRQ_DOMAIN_FLAG_MSI_DEVICE
 parent: 5D202AA8-1E03-4F0F-A786-390A0D2749E9-3
    name:   5D202AA8-1E03-4F0F-A786-390A0D2749E9-3
     size:   0
     mapped: 7
     flags:  0x00000103
                IRQ_DOMAIN_FLAG_HIERARCHY
                IRQ_DOMAIN_NAME_ALLOCATED
                IRQ_DOMAIN_FLAG_MSI_PARENT
     parent: hv_vpci_arm64
        name:   hv_vpci_arm64
         size:   956
         mapped: 31
         flags:  0x00000003
                    IRQ_DOMAIN_FLAG_HIERARCHY
                    IRQ_DOMAIN_NAME_ALLOCATED
         parent: irqchip@0x00000000ffff0000-1
            name:   irqchip@0x00000000ffff0000-1
             size:   0
             mapped: 47
             flags:  0x00000003
                        IRQ_DOMAIN_FLAG_HIERARCHY
                        IRQ_DOMAIN_NAME_ALLOCATED

The 5D202AA8-1E03-4F0F-A786-390A0D2749E9-3 domain has
IRQ_DOMAIN_FLAG_MSI_PARENT set. But the hv_vpci_arm64
and irqchip@... domains do not.  Is that a problem?  On x86,
the output is this, with IRQ_DOMAIN_FLAG_MSI_PARENT set
in the next level up VECTOR domain:

# cat HV-PCI-MSIX-6b71\:00\:02.0-12
name:   HV-PCI-MSIX-6b71:00:02.0-12
 size:   0
 mapped: 17
 flags:  0x00000213
            IRQ_DOMAIN_FLAG_HIERARCHY
            IRQ_DOMAIN_NAME_ALLOCATED
            IRQ_DOMAIN_FLAG_MSI
            IRQ_DOMAIN_FLAG_MSI_DEVICE
 parent: 8564CB14-6B71-477C-B189-F175118E6FF0-3
    name:   8564CB14-6B71-477C-B189-F175118E6FF0-3
     size:   0
     mapped: 17
     flags:  0x00000103
                IRQ_DOMAIN_FLAG_HIERARCHY
                IRQ_DOMAIN_NAME_ALLOCATED
                IRQ_DOMAIN_FLAG_MSI_PARENT
     parent: VECTOR
        name:   VECTOR
         size:   0
         mapped: 67
         flags:  0x00000103
                    IRQ_DOMAIN_FLAG_HIERARCHY
                    IRQ_DOMAIN_NAME_ALLOCATED
                    IRQ_DOMAIN_FLAG_MSI_PARENT

Finally, I've noted a couple of code review comments below. These
comments may reflect my lack of fully understanding the MSI
IRQ handling, in which case, please set me straight. Thanks,

Michael

> 
> Signed-off-by: Nam Cao <namcao@linutronix.de>
> ---
> Cc: K. Y. Srinivasan <kys@microsoft.com>
> Cc: Haiyang Zhang <haiyangz@microsoft.com>
> Cc: Wei Liu <wei.liu@kernel.org>
> Cc: Dexuan Cui <decui@microsoft.com>
> Cc: linux-hyperv@vger.kernel.org
> ---
>  drivers/pci/Kconfig                 |  1 +
>  drivers/pci/controller/pci-hyperv.c | 98 +++++++++++++++++++++++------
>  2 files changed, 80 insertions(+), 19 deletions(-)
> 
> diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
> index 9c0e4aaf4e8cb..9a249c65aedcd 100644
> --- a/drivers/pci/Kconfig
> +++ b/drivers/pci/Kconfig
> @@ -223,6 +223,7 @@ config PCI_HYPERV
>  	tristate "Hyper-V PCI Frontend"
>  	depends on ((X86 && X86_64) || ARM64) && HYPERV && PCI_MSI && SYSFS
>  	select PCI_HYPERV_INTERFACE
> +	select IRQ_MSI_LIB
>  	help
>  	  The PCI device frontend driver allows the kernel to import arbitrary
>  	  PCI devices from a PCI backend to support PCI driver domains.
> diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
> index ef5d655a0052c..3a24fadddb83b 100644
> --- a/drivers/pci/controller/pci-hyperv.c
> +++ b/drivers/pci/controller/pci-hyperv.c
> @@ -44,6 +44,7 @@
>  #include <linux/delay.h>
>  #include <linux/semaphore.h>
>  #include <linux/irq.h>
> +#include <linux/irqchip/irq-msi-lib.h>
>  #include <linux/msi.h>
>  #include <linux/hyperv.h>
>  #include <linux/refcount.h>
> @@ -508,7 +509,6 @@ struct hv_pcibus_device {
>  	struct list_head children;
>  	struct list_head dr_list;
> 
> -	struct msi_domain_info msi_info;
>  	struct irq_domain *irq_domain;
> 
>  	struct workqueue_struct *wq;
> @@ -1687,7 +1687,7 @@ static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
>  	struct msi_desc *msi = irq_data_get_msi_desc(irq_data);
> 
>  	pdev = msi_desc_to_pci_dev(msi);
> -	hbus = info->data;
> +	hbus = domain->host_data;
>  	int_desc = irq_data_get_irq_chip_data(irq_data);
>  	if (!int_desc)
>  		return;
> @@ -1705,7 +1705,6 @@ static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
> 
>  static void hv_irq_mask(struct irq_data *data)
>  {
> -	pci_msi_mask_irq(data);
>  	if (data->parent_data->chip->irq_mask)
>  		irq_chip_mask_parent(data);
>  }
> @@ -1716,7 +1715,6 @@ static void hv_irq_unmask(struct irq_data *data)
> 
>  	if (data->parent_data->chip->irq_unmask)
>  		irq_chip_unmask_parent(data);
> -	pci_msi_unmask_irq(data);
>  }
> 
>  struct compose_comp_ctxt {
> @@ -2101,6 +2099,44 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
>  	msg->data = 0;
>  }
> 
> +static bool hv_pcie_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
> +				      struct irq_domain *real_parent, struct msi_domain_info *info)
> +{
> +	struct irq_chip *chip = info->chip;
> +
> +	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
> +		return false;
> +
> +	info->ops->msi_prepare = hv_msi_prepare;
> +
> +	chip->irq_set_affinity = irq_chip_set_affinity_parent;
> +
> +	if (IS_ENABLED(CONFIG_X86))
> +		chip->flags |= IRQCHIP_MOVE_DEFERRED;
> +
> +	return true;
> +}
> +
> +#define HV_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
> +				    MSI_FLAG_USE_DEF_CHIP_OPS		| \
> +				    MSI_FLAG_PCI_MSI_MASK_PARENT)
> +#define HV_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI	| \
> +				     MSI_FLAG_PCI_MSIX			| \
> +				     MSI_GENERIC_FLAGS_MASK)
> +
> +static const struct msi_parent_ops hv_pcie_msi_parent_ops = {
> +	.required_flags		= HV_PCIE_MSI_FLAGS_REQUIRED,
> +	.supported_flags	= HV_PCIE_MSI_FLAGS_SUPPORTED,
> +	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
> +#ifdef CONFIG_X86
> +	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
> +#elif defined(CONFIG_ARM64)
> +	.chip_flags		= MSI_CHIP_FLAG_SET_EOI,
> +#endif
> +	.prefix			= "HV-",
> +	.init_dev_msi_info	= hv_pcie_init_dev_msi_info,
> +};
> +
>  /* HW Interrupt Chip Descriptor */
>  static struct irq_chip hv_msi_irq_chip = {
>  	.name			= "Hyper-V PCIe MSI",
> @@ -2108,7 +2144,6 @@ static struct irq_chip hv_msi_irq_chip = {
>  	.irq_set_affinity	= irq_chip_set_affinity_parent,
>  #ifdef CONFIG_X86
>  	.irq_ack		= irq_chip_ack_parent,
> -	.flags			= IRQCHIP_MOVE_DEFERRED,
>  #elif defined(CONFIG_ARM64)
>  	.irq_eoi		= irq_chip_eoi_parent,
>  #endif

Would it work to drop the #ifdef's and always set both .irq_ack
and .irq_eoi on x86 and on ARM64? Is which one gets called
controlled by the child HV-PCI-MSIX- ... domain, based on
the .chip_flags? I'm trying to reduce the #ifdef clutter. I
tested without the #ifdefs on both x86 and arm64, and
everything works, but I know that doesn't prove that it's
OK.

If the #ifdefs can go away, then I'd like to see a tweak to the way
.chip_flags is set. Rather than do an #ifdef inline for struct
msi_parent_ops hv_pcie_msi_parent_ops, add a #define
HV_MSI_CHIP_FLAGS in the existing #ifdef X86 and #ifdef ARM64
sections respectively near the top of this source file, and then
use HV_MSI_CHIP_FLAGS in struct msi_parent_ops
hv_pcie_msi_parent_ops.  As much as is reasonable, I'd like to
not clutter the code with #ifdef X86 #elseif ARM64, but instead
group all the differences under the existing #ifdefs near the top.
There are some places where this isn't practical, but this seems
like a place that is practical.

> @@ -2116,9 +2151,37 @@ static struct irq_chip hv_msi_irq_chip = {
>  	.irq_unmask		= hv_irq_unmask,
>  };
> 
> -static struct msi_domain_ops hv_msi_ops = {
> -	.msi_prepare	= hv_msi_prepare,
> -	.msi_free	= hv_msi_free,
> +static int hv_pcie_domain_alloc(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs,
> +			       void *arg)
> +{
> +	/* TODO: move the content of hv_compose_msi_msg() in here */

Could you elaborate on this TODO? Is the idea to loop through all the IRQs and
generate the MSI message for each one? What is the advantage to doing it here?
I noticed in Patch 3 of the series, the Aardvark controller has
advk_msi_irq_compose_msi_msg(), but you had not moved it into the domain
allocation path.

Also, is there some point in the time in the future where the "TODO" is likely to
become a "MUST DO"?

> +	int ret;
> +
> +	ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, arg);
> +	if (ret < 0)
> +		return ret;
> +
> +	for (int i = 0; i < nr_irqs; i++) {
> +		irq_domain_set_info(d, virq + i, 0, &hv_msi_irq_chip, NULL, FLOW_HANDLER, NULL,
> +				    FLOW_NAME);
> +	}
> +
> +	return 0;
> +}
> +
> +static void hv_pcie_domain_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs)
> +{
> +	struct msi_domain_info *info = d->host_data;
> +
> +	for (int i = 0; i < nr_irqs; i++)
> +		hv_msi_free(d, info, virq + i);
> +
> +	irq_domain_free_irqs_top(d, virq, nr_irqs);
> +}
> +
> +static const struct irq_domain_ops hv_pcie_domain_ops = {
> +	.alloc	= hv_pcie_domain_alloc,
> +	.free	= hv_pcie_domain_free,
>  };
> 
>  /**
> @@ -2136,17 +2199,14 @@ static struct msi_domain_ops hv_msi_ops = {
>   */
>  static int hv_pcie_init_irq_domain(struct hv_pcibus_device *hbus)
>  {
> -	hbus->msi_info.chip = &hv_msi_irq_chip;
> -	hbus->msi_info.ops = &hv_msi_ops;
> -	hbus->msi_info.flags = (MSI_FLAG_USE_DEF_DOM_OPS |
> -		MSI_FLAG_USE_DEF_CHIP_OPS | MSI_FLAG_MULTI_PCI_MSI |
> -		MSI_FLAG_PCI_MSIX);
> -	hbus->msi_info.handler = FLOW_HANDLER;
> -	hbus->msi_info.handler_name = FLOW_NAME;
> -	hbus->msi_info.data = hbus;
> -	hbus->irq_domain = pci_msi_create_irq_domain(hbus->fwnode,
> -						     &hbus->msi_info,
> -						     hv_pci_get_root_domain());
> +	struct irq_domain_info info = {
> +		.fwnode		= hbus->fwnode,
> +		.ops		= &hv_pcie_domain_ops,
> +		.host_data	= hbus,
> +		.parent		= hv_pci_get_root_domain(),
> +	};
> +
> +	hbus->irq_domain = msi_create_parent_irq_domain(&info, &hv_pcie_msi_parent_ops);
>  	if (!hbus->irq_domain) {
>  		dev_err(&hbus->hdev->device,
>  			"Failed to build an MSI IRQ domain\n");
> --
> 2.39.5
> 

Re: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Nam Cao]

On Sat, Jul 05, 2025 at 03:51:48AM +0000, Michael Kelley wrote:
> From: Nam Cao <namcao@linutronix.de> Sent: Thursday, June 26, 2025 7:48 AM
> > 
> > Move away from the legacy MSI domain setup, switch to use
> > msi_create_parent_irq_domain().
> 
> With the additional tweak to this patch that you supplied separately,
> everything in my testing on both x86 and arm64 seems to work OK. So
> that's all good.
> 
> On arm64, I did notice the following IRQ domain information from
> /sys/kernel/debug/irq/domains:
> 
> # cat HV-PCI-MSIX-1e03\:00\:00.0-12
> name:   HV-PCI-MSIX-1e03:00:00.0-12
>  size:   0
>  mapped: 7
>  flags:  0x00000213
>             IRQ_DOMAIN_FLAG_HIERARCHY
>             IRQ_DOMAIN_NAME_ALLOCATED
>             IRQ_DOMAIN_FLAG_MSI
>             IRQ_DOMAIN_FLAG_MSI_DEVICE
>  parent: 5D202AA8-1E03-4F0F-A786-390A0D2749E9-3
>     name:   5D202AA8-1E03-4F0F-A786-390A0D2749E9-3
>      size:   0
>      mapped: 7
>      flags:  0x00000103
>                 IRQ_DOMAIN_FLAG_HIERARCHY
>                 IRQ_DOMAIN_NAME_ALLOCATED
>                 IRQ_DOMAIN_FLAG_MSI_PARENT
>      parent: hv_vpci_arm64
>         name:   hv_vpci_arm64
>          size:   956
>          mapped: 31
>          flags:  0x00000003
>                     IRQ_DOMAIN_FLAG_HIERARCHY
>                     IRQ_DOMAIN_NAME_ALLOCATED
>          parent: irqchip@0x00000000ffff0000-1
>             name:   irqchip@0x00000000ffff0000-1
>              size:   0
>              mapped: 47
>              flags:  0x00000003
>                         IRQ_DOMAIN_FLAG_HIERARCHY
>                         IRQ_DOMAIN_NAME_ALLOCATED
> 
> The 5D202AA8-1E03-4F0F-A786-390A0D2749E9-3 domain has
> IRQ_DOMAIN_FLAG_MSI_PARENT set. But the hv_vpci_arm64
> and irqchip@... domains do not.  Is that a problem?  On x86,
> the output is this, with IRQ_DOMAIN_FLAG_MSI_PARENT set
> in the next level up VECTOR domain:

That looks normal. IRQ_DOMAIN_FLAG_MSI_PARENT is set for domains which
provide MSI parent domain capability, which happens to be the case for x86
vector.

> # cat HV-PCI-MSIX-6b71\:00\:02.0-12
> name:   HV-PCI-MSIX-6b71:00:02.0-12
>  size:   0
>  mapped: 17
>  flags:  0x00000213
>             IRQ_DOMAIN_FLAG_HIERARCHY
>             IRQ_DOMAIN_NAME_ALLOCATED
>             IRQ_DOMAIN_FLAG_MSI
>             IRQ_DOMAIN_FLAG_MSI_DEVICE
>  parent: 8564CB14-6B71-477C-B189-F175118E6FF0-3
>     name:   8564CB14-6B71-477C-B189-F175118E6FF0-3
>      size:   0
>      mapped: 17
>      flags:  0x00000103
>                 IRQ_DOMAIN_FLAG_HIERARCHY
>                 IRQ_DOMAIN_NAME_ALLOCATED
>                 IRQ_DOMAIN_FLAG_MSI_PARENT
>      parent: VECTOR
>         name:   VECTOR
>          size:   0
>          mapped: 67
>          flags:  0x00000103
>                     IRQ_DOMAIN_FLAG_HIERARCHY
>                     IRQ_DOMAIN_NAME_ALLOCATED
>                     IRQ_DOMAIN_FLAG_MSI_PARENT
> 
> Finally, I've noted a couple of code review comments below. These
> comments may reflect my lack of fully understanding the MSI
> IRQ handling, in which case, please set me straight. Thanks,
> 
> Michael
> 
> > 
> > Signed-off-by: Nam Cao <namcao@linutronix.de>
> > ---
> > Cc: K. Y. Srinivasan <kys@microsoft.com>
> > Cc: Haiyang Zhang <haiyangz@microsoft.com>
> > Cc: Wei Liu <wei.liu@kernel.org>
> > Cc: Dexuan Cui <decui@microsoft.com>
> > Cc: linux-hyperv@vger.kernel.org
> > ---
> >  drivers/pci/Kconfig                 |  1 +
> >  drivers/pci/controller/pci-hyperv.c | 98 +++++++++++++++++++++++------
> >  2 files changed, 80 insertions(+), 19 deletions(-)
> > 
> > diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
> > index 9c0e4aaf4e8cb..9a249c65aedcd 100644
> > --- a/drivers/pci/Kconfig
> > +++ b/drivers/pci/Kconfig
> > @@ -223,6 +223,7 @@ config PCI_HYPERV
> >  	tristate "Hyper-V PCI Frontend"
> >  	depends on ((X86 && X86_64) || ARM64) && HYPERV && PCI_MSI && SYSFS
> >  	select PCI_HYPERV_INTERFACE
> > +	select IRQ_MSI_LIB
> >  	help
> >  	  The PCI device frontend driver allows the kernel to import arbitrary
> >  	  PCI devices from a PCI backend to support PCI driver domains.
> > diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c
> > index ef5d655a0052c..3a24fadddb83b 100644
> > --- a/drivers/pci/controller/pci-hyperv.c
> > +++ b/drivers/pci/controller/pci-hyperv.c
> > @@ -44,6 +44,7 @@
> >  #include <linux/delay.h>
> >  #include <linux/semaphore.h>
> >  #include <linux/irq.h>
> > +#include <linux/irqchip/irq-msi-lib.h>
> >  #include <linux/msi.h>
> >  #include <linux/hyperv.h>
> >  #include <linux/refcount.h>
> > @@ -508,7 +509,6 @@ struct hv_pcibus_device {
> >  	struct list_head children;
> >  	struct list_head dr_list;
> > 
> > -	struct msi_domain_info msi_info;
> >  	struct irq_domain *irq_domain;
> > 
> >  	struct workqueue_struct *wq;
> > @@ -1687,7 +1687,7 @@ static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
> >  	struct msi_desc *msi = irq_data_get_msi_desc(irq_data);
> > 
> >  	pdev = msi_desc_to_pci_dev(msi);
> > -	hbus = info->data;
> > +	hbus = domain->host_data;
> >  	int_desc = irq_data_get_irq_chip_data(irq_data);
> >  	if (!int_desc)
> >  		return;
> > @@ -1705,7 +1705,6 @@ static void hv_msi_free(struct irq_domain *domain, struct msi_domain_info *info,
> > 
> >  static void hv_irq_mask(struct irq_data *data)
> >  {
> > -	pci_msi_mask_irq(data);
> >  	if (data->parent_data->chip->irq_mask)
> >  		irq_chip_mask_parent(data);
> >  }
> > @@ -1716,7 +1715,6 @@ static void hv_irq_unmask(struct irq_data *data)
> > 
> >  	if (data->parent_data->chip->irq_unmask)
> >  		irq_chip_unmask_parent(data);
> > -	pci_msi_unmask_irq(data);
> >  }
> > 
> >  struct compose_comp_ctxt {
> > @@ -2101,6 +2099,44 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
> >  	msg->data = 0;
> >  }
> > 
> > +static bool hv_pcie_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
> > +				      struct irq_domain *real_parent, struct msi_domain_info *info)
> > +{
> > +	struct irq_chip *chip = info->chip;
> > +
> > +	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
> > +		return false;
> > +
> > +	info->ops->msi_prepare = hv_msi_prepare;
> > +
> > +	chip->irq_set_affinity = irq_chip_set_affinity_parent;
> > +
> > +	if (IS_ENABLED(CONFIG_X86))
> > +		chip->flags |= IRQCHIP_MOVE_DEFERRED;
> > +
> > +	return true;
> > +}
> > +
> > +#define HV_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS	| \
> > +				    MSI_FLAG_USE_DEF_CHIP_OPS		| \
> > +				    MSI_FLAG_PCI_MSI_MASK_PARENT)
> > +#define HV_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI	| \
> > +				     MSI_FLAG_PCI_MSIX			| \
> > +				     MSI_GENERIC_FLAGS_MASK)
> > +
> > +static const struct msi_parent_ops hv_pcie_msi_parent_ops = {
> > +	.required_flags		= HV_PCIE_MSI_FLAGS_REQUIRED,
> > +	.supported_flags	= HV_PCIE_MSI_FLAGS_SUPPORTED,
> > +	.bus_select_token	= DOMAIN_BUS_PCI_MSI,
> > +#ifdef CONFIG_X86
> > +	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
> > +#elif defined(CONFIG_ARM64)
> > +	.chip_flags		= MSI_CHIP_FLAG_SET_EOI,
> > +#endif
> > +	.prefix			= "HV-",
> > +	.init_dev_msi_info	= hv_pcie_init_dev_msi_info,
> > +};
> > +
> >  /* HW Interrupt Chip Descriptor */
> >  static struct irq_chip hv_msi_irq_chip = {
> >  	.name			= "Hyper-V PCIe MSI",
> > @@ -2108,7 +2144,6 @@ static struct irq_chip hv_msi_irq_chip = {
> >  	.irq_set_affinity	= irq_chip_set_affinity_parent,
> >  #ifdef CONFIG_X86
> >  	.irq_ack		= irq_chip_ack_parent,
> > -	.flags			= IRQCHIP_MOVE_DEFERRED,
> >  #elif defined(CONFIG_ARM64)
> >  	.irq_eoi		= irq_chip_eoi_parent,
> >  #endif
> 
> Would it work to drop the #ifdef's and always set both .irq_ack and
> .irq_eoi on x86 and on ARM64?  Is which one gets called controlled by the
> child HV-PCI-MSIX- ... domain, based on the .chip_flags?
>
> I'm trying to reduce the #ifdef clutter. I
> tested without the #ifdefs on both x86 and arm64, and
> everything works, but I know that doesn't prove that it's
> OK.

Nothing is wrong with that, as far as I can tell.

> If the #ifdefs can go away, then I'd like to see a tweak to the way
> .chip_flags is set. Rather than do an #ifdef inline for struct
> msi_parent_ops hv_pcie_msi_parent_ops, add a #define
> HV_MSI_CHIP_FLAGS in the existing #ifdef X86 and #ifdef ARM64
> sections respectively near the top of this source file, and then
> use HV_MSI_CHIP_FLAGS in struct msi_parent_ops
> hv_pcie_msi_parent_ops.  As much as is reasonable, I'd like to
> not clutter the code with #ifdef X86 #elseif ARM64, but instead
> group all the differences under the existing #ifdefs near the top.
> There are some places where this isn't practical, but this seems
> like a place that is practical.

Yes, that would be better. I will do it in v2.

> > @@ -2116,9 +2151,37 @@ static struct irq_chip hv_msi_irq_chip = {
> >  	.irq_unmask		= hv_irq_unmask,
> >  };
> > 
> > -static struct msi_domain_ops hv_msi_ops = {
> > -	.msi_prepare	= hv_msi_prepare,
> > -	.msi_free	= hv_msi_free,
> > +static int hv_pcie_domain_alloc(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs,
> > +			       void *arg)
> > +{
> > +	/* TODO: move the content of hv_compose_msi_msg() in here */
> 
> Could you elaborate on this TODO? Is the idea to loop through all the IRQs and
> generate the MSI message for each one? What is the advantage to doing it here?
> I noticed in Patch 3 of the series, the Aardvark controller has
> advk_msi_irq_compose_msi_msg(), but you had not moved it into the domain
> allocation path.

Sorry for being unclear. hv_compose_msi_msg() should not be moved here
entirely. Let me elaborate this in v2.

What I meant is that, hv_compose_msi_msg() is doing more than what this
callback is supposed to do (composing message). It works, but it is not
correct. Interrupt allocation is the responsibility of
irq_domain_ops::alloc(). Allocating and populating int_desc should be in
hv_pcie_domain_alloc() instead.

irq_domain_ops's .alloc() and .free() should be asymmetric.

> 
> Also, is there some point in the time in the future where the "TODO" is likely to
> become a "MUST DO"?

There's nothing planned that would make this non-functional, as far as I
know.

Thanks so much for examining the patch,
Nam

Re: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Nam Cao]

On Sat, Jul 05, 2025 at 11:46:59AM +0200, Nam Cao wrote:
> On Sat, Jul 05, 2025 at 03:51:48AM +0000, Michael Kelley wrote:
> > From: Nam Cao <namcao@linutronix.de> Sent: Thursday, June 26, 2025 7:48 AM
> > > 
> > > Move away from the legacy MSI domain setup, switch to use
> > > msi_create_parent_irq_domain().
> > 
> > With the additional tweak to this patch that you supplied separately,
> > everything in my testing on both x86 and arm64 seems to work OK. So
> > that's all good.
>
> Thanks so much for examining the patch,

Btw, you probably would also be interested in
https://lore.kernel.org/lkml/cover.1751277765.git.namcao@linutronix.de/

which does a similar conversion for the other hyperv driver.

Nam

Re: [PATCH 00/16] PCI: MSI parent domain conversion

[Manivannan Sadhasivam]

On Fri, Jul 04, 2025 at 06:48:06AM GMT, Nam Cao wrote:
> On Thu, Jul 03, 2025 at 12:28:01PM -0500, Bjorn Helgaas wrote:
> > On Thu, Jun 26, 2025 at 04:47:50PM +0200, Nam Cao wrote:
> > > The initial implementation of PCI/MSI interrupt domains in the hierarchical
> > > interrupt domain model used a shortcut by providing a global PCI/MSI
> > > domain.
> > > 
> > > This works because the PCI/MSI[X] hardware is standardized and uniform, but
> > > it violates the basic design principle of hierarchical interrupt domains:
> > > Each hardware block involved in the interrupt delivery chain should have a
> > > separate interrupt domain.
> > > 
> > > For PCI/MSI[X], the interrupt controller is per PCI device and not a global
> > > made-up entity.
> > > 
> > > Unsurprisingly, the shortcut turned out to have downsides as it does not
> > > allow dynamic allocation of interrupt vectors after initialization and it
> > > prevents supporting IMS on PCI. For further details, see:
> > > 
> > > https://lore.kernel.org/lkml/20221111120501.026511281@linutronix.de/
> > > 
> > > The solution is implementing per device MSI domains, this means the
> > > entities which provide global PCI/MSI domain so far have to implement MSI
> > > parent domain functionality instead.
> > > 
> > > This series converts the PCI controller drivers to implement MSI parent
> > > domain.
> > > 
> > >  drivers/pci/Kconfig                           |   1 +
> > >  drivers/pci/controller/Kconfig                |  11 +
> > >  drivers/pci/controller/dwc/Kconfig            |   1 +
> > >  .../pci/controller/dwc/pcie-designware-host.c |  68 ++----
> > >  drivers/pci/controller/dwc/pcie-designware.h  |   1 -
> > >  drivers/pci/controller/mobiveil/Kconfig       |   1 +
> > >  .../controller/mobiveil/pcie-mobiveil-host.c  |  42 ++--
> > >  .../pci/controller/mobiveil/pcie-mobiveil.h   |   1 -
> > >  drivers/pci/controller/pci-aardvark.c         |  59 ++---
> > >  drivers/pci/controller/pci-hyperv.c           |  98 ++++++--
> > >  drivers/pci/controller/pcie-altera-msi.c      |  44 ++--
> > >  drivers/pci/controller/pcie-brcmstb.c         |  44 ++--
> > >  drivers/pci/controller/pcie-iproc-msi.c       |  45 ++--
> > >  drivers/pci/controller/pcie-mediatek-gen3.c   |  67 ++---
> > >  drivers/pci/controller/pcie-mediatek.c        |  46 ++--
> > >  drivers/pci/controller/pcie-rcar-host.c       |  69 ++----
> > >  drivers/pci/controller/pcie-xilinx-dma-pl.c   |  48 ++--
> > >  drivers/pci/controller/pcie-xilinx-nwl.c      |  45 ++--
> > >  drivers/pci/controller/pcie-xilinx.c          |  55 +++--
> > >  drivers/pci/controller/plda/Kconfig           |   1 +
> > >  drivers/pci/controller/plda/pcie-plda-host.c  |  44 ++--
> > >  drivers/pci/controller/plda/pcie-plda.h       |   1 -
> > >  drivers/pci/controller/vmd.c                  | 229 +++++++++---------
> > >  23 files changed, 504 insertions(+), 517 deletions(-)
> > 
> > Looks good to me, thanks!  I think Mani will probably pick this up.
> > 
> > I might have included the specific "legacy MSI domain" thing you're
> > replacing.  It looks like you're replacing pci_msi_create_irq_domain()
> > with msi_create_parent_irq_domain()?
> 
> Yes, pci_msi_create_irq_domain() is legacy. We will delete it once
> everything is converted.
> 

I'll amend the commit message to include pci_msi_create_irq_domain().

> > Minor merge conflict in pcie-mediatek-gen3.c with dcbea1c7e94e ("PCI:
> > mediatek-gen3: Use dev_fwnode() for irq_domain_create_linear()").  No
> > problem, we can easily fix that up.
> 
> Thanks!
> 
> > The "++i" in vmd.c stuck out to me since "i++" is so much more common.
> 
> I always do "++i", maybe I'm the weird one..
> 

Just for the sake of uniformity, I'll do 's/++i/i++' while applying.

And based on the discussion on patch 14/16, I'll drop it to be applied through
netdev and apply the rest of the series through PCI tree.

- Mani

-- 
மணிவண்ணன் சதாசிவம்

Re: [PATCH 00/16] PCI: MSI parent domain conversion

[Manivannan Sadhasivam]

On Thu, Jun 26, 2025 at 04:47:50PM GMT, Nam Cao wrote:
> The initial implementation of PCI/MSI interrupt domains in the hierarchical
> interrupt domain model used a shortcut by providing a global PCI/MSI
> domain.
> 
> This works because the PCI/MSI[X] hardware is standardized and uniform, but
> it violates the basic design principle of hierarchical interrupt domains:
> Each hardware block involved in the interrupt delivery chain should have a
> separate interrupt domain.
> 
> For PCI/MSI[X], the interrupt controller is per PCI device and not a global
> made-up entity.
> 
> Unsurprisingly, the shortcut turned out to have downsides as it does not
> allow dynamic allocation of interrupt vectors after initialization and it
> prevents supporting IMS on PCI. For further details, see:
> 
> https://lore.kernel.org/lkml/20221111120501.026511281@linutronix.de/
> 
> The solution is implementing per device MSI domains, this means the
> entities which provide global PCI/MSI domain so far have to implement MSI
> parent domain functionality instead.
> 
> This series converts the PCI controller drivers to implement MSI parent
> domain.
> 

Applied the series except patch 14/16 to pci/controller/msi-parent, thanks!

- Mani

>  drivers/pci/Kconfig                           |   1 +
>  drivers/pci/controller/Kconfig                |  11 +
>  drivers/pci/controller/dwc/Kconfig            |   1 +
>  .../pci/controller/dwc/pcie-designware-host.c |  68 ++----
>  drivers/pci/controller/dwc/pcie-designware.h  |   1 -
>  drivers/pci/controller/mobiveil/Kconfig       |   1 +
>  .../controller/mobiveil/pcie-mobiveil-host.c  |  42 ++--
>  .../pci/controller/mobiveil/pcie-mobiveil.h   |   1 -
>  drivers/pci/controller/pci-aardvark.c         |  59 ++---
>  drivers/pci/controller/pci-hyperv.c           |  98 ++++++--
>  drivers/pci/controller/pcie-altera-msi.c      |  44 ++--
>  drivers/pci/controller/pcie-brcmstb.c         |  44 ++--
>  drivers/pci/controller/pcie-iproc-msi.c       |  45 ++--
>  drivers/pci/controller/pcie-mediatek-gen3.c   |  67 ++---
>  drivers/pci/controller/pcie-mediatek.c        |  46 ++--
>  drivers/pci/controller/pcie-rcar-host.c       |  69 ++----
>  drivers/pci/controller/pcie-xilinx-dma-pl.c   |  48 ++--
>  drivers/pci/controller/pcie-xilinx-nwl.c      |  45 ++--
>  drivers/pci/controller/pcie-xilinx.c          |  55 +++--
>  drivers/pci/controller/plda/Kconfig           |   1 +
>  drivers/pci/controller/plda/pcie-plda-host.c  |  44 ++--
>  drivers/pci/controller/plda/pcie-plda.h       |   1 -
>  drivers/pci/controller/vmd.c                  | 229 +++++++++---------
>  23 files changed, 504 insertions(+), 517 deletions(-)
> 
> -- 
> 2.39.5
> 

-- 
மணிவண்ணன் சதாசிவம்

RE: [PATCH 14/16] PCI: hv: Switch to msi_create_parent_irq_domain()

[Michael Kelley]

From: Nam Cao <namcao@linutronix.de> Sent: Saturday, July 5, 2025 3:02 AM
> 
> On Sat, Jul 05, 2025 at 11:46:59AM +0200, Nam Cao wrote:
> > On Sat, Jul 05, 2025 at 03:51:48AM +0000, Michael Kelley wrote:
> > > From: Nam Cao <namcao@linutronix.de> Sent: Thursday, June 26, 2025 7:48 AM
> > > >
> > > > Move away from the legacy MSI domain setup, switch to use
> > > > msi_create_parent_irq_domain().
> > >
> > > With the additional tweak to this patch that you supplied separately,
> > > everything in my testing on both x86 and arm64 seems to work OK. So
> > > that's all good.
> >
> > Thanks so much for examining the patch,
> 
> Btw, you probably would also be interested in
> https://lore.kernel.org/lkml/cover.1751277765.git.namcao@linutronix.de/
> 
> which does a similar conversion for the other hyperv driver.
> 

Thanks. I had seen this other small series for the driver used by Linux
when running as the root partition (i.e., "dom0") on the Hyper-V
hypervisor. Unfortunately, I have less familiarity with that configuration,
and I lack access to hardware needed to run and test it. Someone at
Microsoft would need to look at the patches and give them a basic smoke
test with Linux-as-root.

FWIW, I was a Microsoft employee for many years, working on Linux
guests on Hyper-V. But I retired about 2 years ago. Though I've kept my
hands in Linux kernel work, my access to hardware configs is more
limited than when I was an employee -- now it's guest VMs on my
personal laptop and Azure VMs.

Michael

Re: [PATCH 16/16] PCI: vmd: Switch to msi_create_parent_irq_domain()

[Nirmal Patel]

On Thu, 26 Jun 2025 16:48:06 +0200
Nam Cao <namcao@linutronix.de> wrote:

> Move away from the legacy MSI domain setup, switch to use
> msi_create_parent_irq_domain().
> 
> Signed-off-by: Nam Cao <namcao@linutronix.de>
> ---
> Cc: Nirmal Patel <nirmal.patel@linux.intel.com>
> Cc: Jonathan Derrick <jonathan.derrick@linux.dev>
> ---
>  drivers/pci/controller/Kconfig |   1 +
>  drivers/pci/controller/vmd.c   | 160
> +++++++++++++++++---------------- 2 files changed, 82 insertions(+),
> 79 deletions(-)
> 
> diff --git a/drivers/pci/controller/Kconfig
> b/drivers/pci/controller/Kconfig index 8f56ffd029ba2..41748d083b933
> 100644 --- a/drivers/pci/controller/Kconfig
> +++ b/drivers/pci/controller/Kconfig
> @@ -156,6 +156,7 @@ config PCI_IXP4XX
>  config VMD
>  	depends on PCI_MSI && X86_64 && !UML
>  	tristate "Intel Volume Management Device Driver"
> +	select IRQ_MSI_LIB
>  	help
>  	  Adds support for the Intel Volume Management Device (VMD).
> VMD is a secondary PCI host bridge that allows PCI Express root ports,
> diff --git a/drivers/pci/controller/vmd.c
> b/drivers/pci/controller/vmd.c index d9b893bf4e456..38693a9487d9b
> 100644 --- a/drivers/pci/controller/vmd.c
> +++ b/drivers/pci/controller/vmd.c
> @@ -7,6 +7,7 @@
>  #include <linux/device.h>
>  #include <linux/interrupt.h>
>  #include <linux/irq.h>
> +#include <linux/irqchip/irq-msi-lib.h>
>  #include <linux/kernel.h>
>  #include <linux/module.h>
>  #include <linux/msi.h>
> @@ -174,9 +175,6 @@ static void vmd_compose_msi_msg(struct irq_data
> *data, struct msi_msg *msg) msg->arch_addr_lo.destid_0_7 =
> index_from_irqs(vmd, irq); }
>  
> -/*
> - * We rely on MSI_FLAG_USE_DEF_CHIP_OPS to set the IRQ mask/unmask
> ops.
> - */
>  static void vmd_irq_enable(struct irq_data *data)
>  {
>  	struct vmd_irq *vmdirq = data->chip_data;
> @@ -186,7 +184,11 @@ static void vmd_irq_enable(struct irq_data *data)
>  		list_add_tail_rcu(&vmdirq->node,
> &vmdirq->irq->irq_list); vmdirq->enabled = true;
>  	}
> +}
>  
> +static void vmd_pci_msi_enable(struct irq_data *data)
> +{
> +	vmd_irq_enable(data->parent_data);
>  	data->chip->irq_unmask(data);
>  }
>  
> @@ -194,8 +196,6 @@ static void vmd_irq_disable(struct irq_data *data)
>  {
>  	struct vmd_irq *vmdirq = data->chip_data;
>  
> -	data->chip->irq_mask(data);
> -
>  	scoped_guard(raw_spinlock_irqsave, &list_lock) {
>  		if (vmdirq->enabled) {
>  			list_del_rcu(&vmdirq->node);
> @@ -204,19 +204,17 @@ static void vmd_irq_disable(struct irq_data
> *data) }
>  }
>  
> +static void vmd_pci_msi_disable(struct irq_data *data)
> +{
> +	data->chip->irq_mask(data);
> +	vmd_irq_disable(data->parent_data);
> +}
> +
>  static struct irq_chip vmd_msi_controller = {
>  	.name			= "VMD-MSI",
> -	.irq_enable		= vmd_irq_enable,
> -	.irq_disable		= vmd_irq_disable,
>  	.irq_compose_msi_msg	= vmd_compose_msi_msg,
>  };
>  
> -static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info,
> -				     msi_alloc_info_t *arg)
> -{
> -	return 0;
> -}
> -
>  /*
>   * XXX: We can be even smarter selecting the best IRQ once we solve
> the
>   * affinity problem.
> @@ -250,100 +248,110 @@ static struct vmd_irq_list
> *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *d return
> &vmd->irqs[best]; }
>  
> -static int vmd_msi_init(struct irq_domain *domain, struct
> msi_domain_info *info,
> -			unsigned int virq, irq_hw_number_t hwirq,
> -			msi_alloc_info_t *arg)
> +static void vmd_msi_free(struct irq_domain *domain, unsigned int
> virq, unsigned int nr_irqs); +
> +static int vmd_msi_alloc(struct irq_domain *domain, unsigned int
> virq, unsigned int nr_irqs,
> +			 void *arg)

Is this wrapped in 80 columns? I can see few lines are more than 80.
Disregard this if it is wrapped and it can be my claws mail client
issue.

>  {
> -	struct msi_desc *desc = arg->desc;
> -	struct vmd_dev *vmd =
> vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);
> -	struct vmd_irq *vmdirq = kzalloc(sizeof(*vmdirq),
> GFP_KERNEL);
> +	struct msi_desc *desc = ((msi_alloc_info_t *)arg)->desc;
> +	struct vmd_dev *vmd = domain->host_data;
> +	struct vmd_irq *vmdirq;
>  
> -	if (!vmdirq)
> -		return -ENOMEM;
> +	for (int i = 0; i < nr_irqs; ++i) {
> +		vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL);
> +		if (!vmdirq) {
> +			vmd_msi_free(domain, virq, i);
> +			return -ENOMEM;
> +		}
>  
> -	INIT_LIST_HEAD(&vmdirq->node);
> -	vmdirq->irq = vmd_next_irq(vmd, desc);
> -	vmdirq->virq = virq;
> +		INIT_LIST_HEAD(&vmdirq->node);
> +		vmdirq->irq = vmd_next_irq(vmd, desc);
> +		vmdirq->virq = virq + i;
> +
> +		irq_domain_set_info(domain, virq + i,
> vmdirq->irq->virq, &vmd_msi_controller,
> +				    vmdirq, handle_untracked_irq,
> vmd, NULL);
> +	}
>  
> -	irq_domain_set_info(domain, virq, vmdirq->irq->virq,
> info->chip, vmdirq,
> -			    handle_untracked_irq, vmd, NULL);
>  	return 0;
>  }
>  
> -static void vmd_msi_free(struct irq_domain *domain,
> -			struct msi_domain_info *info, unsigned int
> virq) +static void vmd_msi_free(struct irq_domain *domain, unsigned
> int virq, unsigned int nr_irqs) {
>  	struct vmd_irq *vmdirq = irq_get_chip_data(virq);
>  
> -	synchronize_srcu(&vmdirq->irq->srcu);
> +	for (int i = 0; i < nr_irqs; ++i) {
> +		synchronize_srcu(&vmdirq->irq->srcu);
>  
> -	/* XXX: Potential optimization to rebalance */
> -	scoped_guard(raw_spinlock_irq, &list_lock)
> -		vmdirq->irq->count--;
> +		/* XXX: Potential optimization to rebalance */
> +		scoped_guard(raw_spinlock_irq, &list_lock)
> +			vmdirq->irq->count--;
>  
> -	kfree(vmdirq);
> +		kfree(vmdirq);
> +	}
>  }
>  
> -static int vmd_msi_prepare(struct irq_domain *domain, struct device
> *dev,
> -			   int nvec, msi_alloc_info_t *arg)
> +static const struct irq_domain_ops vmd_msi_domain_ops = {
> +	.alloc		= vmd_msi_alloc,
> +	.free		= vmd_msi_free,
> +};
> +
> +static bool vmd_init_dev_msi_info(struct device *dev, struct
> irq_domain *domain,
> +				  struct irq_domain *real_parent,
> struct msi_domain_info *info) {
> -	struct pci_dev *pdev = to_pci_dev(dev);
> -	struct vmd_dev *vmd = vmd_from_bus(pdev->bus);
> +	if (WARN_ON_ONCE(info->bus_token !=
> DOMAIN_BUS_PCI_DEVICE_MSIX))
> +		return false;
>  
> -	if (nvec > vmd->msix_count)
> -		return vmd->msix_count;
> +	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent,
> info))
> +		return false;
>  
> -	memset(arg, 0, sizeof(*arg));
> -	return 0;
> +	info->chip->irq_enable		= vmd_pci_msi_enable;
> +	info->chip->irq_disable		= vmd_pci_msi_disable;
> +	return true;
>  }
>  
> -static void vmd_set_desc(msi_alloc_info_t *arg, struct msi_desc
> *desc) -{
> -	arg->desc = desc;
> -}
> +#define VMD_MSI_FLAGS_SUPPORTED
> (MSI_GENERIC_FLAGS_MASK | MSI_FLAG_PCI_MSIX) +#define
> VMD_MSI_FLAGS_REQUIRED		(MSI_FLAG_USE_DEF_DOM_OPS |
> MSI_FLAG_NO_AFFINITY) -static struct msi_domain_ops
> vmd_msi_domain_ops = {
> -	.get_hwirq	= vmd_get_hwirq,
> -	.msi_init	= vmd_msi_init,
> -	.msi_free	= vmd_msi_free,
> -	.msi_prepare	= vmd_msi_prepare,
> -	.set_desc	= vmd_set_desc,
> +static const struct msi_parent_ops vmd_msi_parent_ops = {
> +	.supported_flags	= VMD_MSI_FLAGS_SUPPORTED,
> +	.required_flags		= VMD_MSI_FLAGS_REQUIRED,
> +	.bus_select_token	= DOMAIN_BUS_VMD_MSI,
> +	.bus_select_mask	= MATCH_PCI_MSI,
> +	.prefix			= "VMD-",
> +	.init_dev_msi_info	= vmd_init_dev_msi_info,
>  };
>  
> -static struct msi_domain_info vmd_msi_domain_info = {
> -	.flags		= MSI_FLAG_USE_DEF_DOM_OPS |
> MSI_FLAG_USE_DEF_CHIP_OPS |
> -			  MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX,
> -	.ops		= &vmd_msi_domain_ops,
> -	.chip		= &vmd_msi_controller,
> -};
> -
> -static void vmd_set_msi_remapping(struct vmd_dev *vmd, bool enable)
> -{
> -	u16 reg;
> -
> -	pci_read_config_word(vmd->dev, PCI_REG_VMCONFIG, &reg);
> -	reg = enable ? (reg & ~VMCONFIG_MSI_REMAP) :
> -		       (reg | VMCONFIG_MSI_REMAP);
> -	pci_write_config_word(vmd->dev, PCI_REG_VMCONFIG, reg);
> -}
> -
>  static int vmd_create_irq_domain(struct vmd_dev *vmd)
>  {
> -	struct fwnode_handle *fn;
> +	struct irq_domain_info info = {
> +		.size		= vmd->msix_count,
> +		.ops		= &vmd_msi_domain_ops,
> +		.host_data	= vmd,
> +	};
>  
> -	fn = irq_domain_alloc_named_id_fwnode("VMD-MSI",
> vmd->sysdata.domain);
> -	if (!fn)
> +	info.fwnode = irq_domain_alloc_named_id_fwnode("VMD-MSI",
> vmd->sysdata.domain);
> +	if (!info.fwnode)
>  		return -ENODEV;
>  
> -	vmd->irq_domain = pci_msi_create_irq_domain(fn,
> &vmd_msi_domain_info, NULL);
> +	vmd->irq_domain = msi_create_parent_irq_domain(&info,
> &vmd_msi_parent_ops); if (!vmd->irq_domain) {
> -		irq_domain_free_fwnode(fn);
> +		irq_domain_free_fwnode(info.fwnode);
>  		return -ENODEV;
>  	}
>  
>  	return 0;
>  }
>  
> +static void vmd_set_msi_remapping(struct vmd_dev *vmd, bool enable)
> +{
> +	u16 reg;
> +
> +	pci_read_config_word(vmd->dev, PCI_REG_VMCONFIG, &reg);
> +	reg = enable ? (reg & ~VMCONFIG_MSI_REMAP) :
> +		       (reg | VMCONFIG_MSI_REMAP);
> +	pci_write_config_word(vmd->dev, PCI_REG_VMCONFIG, reg);
> +}
> +
>  static void vmd_remove_irq_domain(struct vmd_dev *vmd)
>  {
>  	/*
> @@ -874,12 +882,6 @@ static int vmd_enable_domain(struct vmd_dev
> *vmd, unsigned long features) ret = vmd_create_irq_domain(vmd);
>  		if (ret)
>  			return ret;
> -
> -		/*
> -		 * Override the IRQ domain bus token so the domain
> can be
> -		 * distinguished from a regular PCI/MSI domain.
> -		 */
> -		irq_domain_update_bus_token(vmd->irq_domain,
> DOMAIN_BUS_VMD_MSI); } else {
>  		vmd_set_msi_remapping(vmd, false);
>  	}

Re: [PATCH 16/16] PCI: vmd: Switch to msi_create_parent_irq_domain()

[Bjorn Helgaas]

On Wed, Jul 16, 2025 at 11:10:09AM -0700, Nirmal Patel wrote:
> On Thu, 26 Jun 2025 16:48:06 +0200
> Nam Cao <namcao@linutronix.de> wrote:
> 
> > Move away from the legacy MSI domain setup, switch to use
> > msi_create_parent_irq_domain().

> > -			unsigned int virq, irq_hw_number_t hwirq,
> > -			msi_alloc_info_t *arg)
> > +static void vmd_msi_free(struct irq_domain *domain, unsigned int
> > virq, unsigned int nr_irqs); +
> > +static int vmd_msi_alloc(struct irq_domain *domain, unsigned int
> > virq, unsigned int nr_irqs,
> > +			 void *arg)
> 
> Is this wrapped in 80 columns? I can see few lines are more than 80.
> Disregard this if it is wrapped and it can be my claws mail client
> issue.

Wrapped locally, thanks Nirmal.

Bjorn

Re: [PATCH 16/16] PCI: vmd: Switch to msi_create_parent_irq_domain()

[Antonio Quartulli]

Hi Nam,

On 26/06/2025 16:48, Nam Cao wrote:
[...]
> -static void vmd_msi_free(struct irq_domain *domain,
> -			struct msi_domain_info *info, unsigned int virq)
> +static void vmd_msi_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs)
>   {
>   	struct vmd_irq *vmdirq = irq_get_chip_data(virq);
>   
> -	synchronize_srcu(&vmdirq->irq->srcu);
> +	for (int i = 0; i < nr_irqs; ++i) {
> +		synchronize_srcu(&vmdirq->irq->srcu);
>   
> -	/* XXX: Potential optimization to rebalance */
> -	scoped_guard(raw_spinlock_irq, &list_lock)
> -		vmdirq->irq->count--;
> +		/* XXX: Potential optimization to rebalance */
> +		scoped_guard(raw_spinlock_irq, &list_lock)
> +			vmdirq->irq->count--;
>   
> -	kfree(vmdirq);
> +		kfree(vmdirq);
> +	}

By introducing a for loop in this function, you are re-using vmdirq 
after free'ing it.

I can't send a patch because I am not faimliar with this API and I don't 
know how to fix it.

However, the issue was reported today by Coverity.

Any idea? :-)

Regards,

-- 
Antonio Quartulli

CEO and Co-Founder
Mandelbit Srl
https://www.mandelbit.com

Re: [PATCH 16/16] PCI: vmd: Switch to msi_create_parent_irq_domain()

[Nam Cao]

On Wed, Jul 16, 2025 at 09:52:05PM +0200, Antonio Quartulli wrote:
> Hi Nam,
Hi Antonio,

> On 26/06/2025 16:48, Nam Cao wrote:
> [...]
> > -static void vmd_msi_free(struct irq_domain *domain,
> > -			struct msi_domain_info *info, unsigned int virq)
> > +static void vmd_msi_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs)
> >   {
> >   	struct vmd_irq *vmdirq = irq_get_chip_data(virq);
> > -	synchronize_srcu(&vmdirq->irq->srcu);
> > +	for (int i = 0; i < nr_irqs; ++i) {
> > +		synchronize_srcu(&vmdirq->irq->srcu);
> > -	/* XXX: Potential optimization to rebalance */
> > -	scoped_guard(raw_spinlock_irq, &list_lock)
> > -		vmdirq->irq->count--;
> > +		/* XXX: Potential optimization to rebalance */
> > +		scoped_guard(raw_spinlock_irq, &list_lock)
> > +			vmdirq->irq->count--;
> > -	kfree(vmdirq);
> > +		kfree(vmdirq);
> > +	}
> 
> By introducing a for loop in this function, you are re-using vmdirq after
> free'ing it.
> 
> I can't send a patch because I am not faimliar with this API and I don't
> know how to fix it.
> 
> However, the issue was reported today by Coverity.
> 
> Any idea? :-)

Thanks for the report. That was indeed a mistake from my side.

I hope PCI maintainers don't mind squashing the below diff.

Sorry for the troubles so far,
Nam

diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
index 48a6096cbbc0..50f0c91d561c 100644
--- a/drivers/pci/controller/vmd.c
+++ b/drivers/pci/controller/vmd.c
@@ -280,9 +280,11 @@ static int vmd_msi_alloc(struct irq_domain *domain, unsigned int virq,
 static void vmd_msi_free(struct irq_domain *domain, unsigned int virq,
 			 unsigned int nr_irqs)
 {
-	struct vmd_irq *vmdirq = irq_get_chip_data(virq);
+	struct vmd_irq *vmdirq;
 
 	for (int i = 0; i < nr_irqs; ++i) {
+		vmdirq = irq_get_chip_data(virq + i);
+
 		synchronize_srcu(&vmdirq->irq->srcu);
 
 		/* XXX: Potential optimization to rebalance */

Re: [PATCH 16/16] PCI: vmd: Switch to msi_create_parent_irq_domain()

[Bjorn Helgaas]

On Wed, Jul 16, 2025 at 10:12:16PM +0200, Nam Cao wrote:
> On Wed, Jul 16, 2025 at 09:52:05PM +0200, Antonio Quartulli wrote:
> > Hi Nam,
> Hi Antonio,
> 
> > On 26/06/2025 16:48, Nam Cao wrote:
> > [...]
> > > -static void vmd_msi_free(struct irq_domain *domain,
> > > -			struct msi_domain_info *info, unsigned int virq)
> > > +static void vmd_msi_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs)
> > >   {
> > >   	struct vmd_irq *vmdirq = irq_get_chip_data(virq);
> > > -	synchronize_srcu(&vmdirq->irq->srcu);
> > > +	for (int i = 0; i < nr_irqs; ++i) {
> > > +		synchronize_srcu(&vmdirq->irq->srcu);
> > > -	/* XXX: Potential optimization to rebalance */
> > > -	scoped_guard(raw_spinlock_irq, &list_lock)
> > > -		vmdirq->irq->count--;
> > > +		/* XXX: Potential optimization to rebalance */
> > > +		scoped_guard(raw_spinlock_irq, &list_lock)
> > > +			vmdirq->irq->count--;
> > > -	kfree(vmdirq);
> > > +		kfree(vmdirq);
> > > +	}
> > 
> > By introducing a for loop in this function, you are re-using vmdirq after
> > free'ing it.
> > 
> > I can't send a patch because I am not faimliar with this API and I don't
> > know how to fix it.
> > 
> > However, the issue was reported today by Coverity.
> > 
> > Any idea? :-)
> 
> Thanks for the report. That was indeed a mistake from my side.
> 
> I hope PCI maintainers don't mind squashing the below diff.

Squashed, thanks!  Updated commit:

  https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git/commit/?id=4246b7fccf26

> diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c
> index 48a6096cbbc0..50f0c91d561c 100644
> --- a/drivers/pci/controller/vmd.c
> +++ b/drivers/pci/controller/vmd.c
> @@ -280,9 +280,11 @@ static int vmd_msi_alloc(struct irq_domain *domain, unsigned int virq,
>  static void vmd_msi_free(struct irq_domain *domain, unsigned int virq,
>  			 unsigned int nr_irqs)
>  {
> -	struct vmd_irq *vmdirq = irq_get_chip_data(virq);
> +	struct vmd_irq *vmdirq;
>  
>  	for (int i = 0; i < nr_irqs; ++i) {
> +		vmdirq = irq_get_chip_data(virq + i);
> +
>  		synchronize_srcu(&vmdirq->irq->srcu);
>  
>  		/* XXX: Potential optimization to rebalance */
>