[PATCH v5 0/5] Designware host multivector MSI and 64bit MSI fixes

[PATCH v5 0/5] Designware host multivector MSI and 64bit MSI fixes

[Lucas Stach]

This is a reworked version of the multivector MSI support for the designware
PCIe host controller driver.

v3 added patch 2 to share more code between the single and multivector MSI
code paths.

While not really related to this topic patches 4+5 fix the MSI message setup
to work on 64bit and 32bit PAE systems. I included them in this series as
they depend on patch 2.

v4 fixes the last patch to also set up the high part of the MSI target
address, when calling the common DW MSI init function.

v5 fixes up clashes with the PCI and MSI changes in v4.3-rc1 and fixes
one build failure introduced in the last version.

Regards,
Lucas

Lucas Stach (5):
  PCI: Add msi_controller setup_irqs() method for special multivector
    setup
  PCI: designware: Factor out MSI msg setup
  PCI: designware: Implement multivector MSI IRQ setup
  PCI: designware: Make get_msi_addr() return phys_addr_t, not u32
  PCI: designware: Set up high part of MSI target address

 drivers/pci/host/pci-keystone-dw.c |  2 +-
 drivers/pci/host/pci-keystone.h    |  2 +-
 drivers/pci/host/pcie-designware.c | 70 +++++++++++++++++++++++++++++++-------
 drivers/pci/host/pcie-designware.h |  2 +-
 drivers/pci/msi.c                  |  3 ++
 include/linux/msi.h                |  2 ++
 6 files changed, 65 insertions(+), 16 deletions(-)

-- 
2.5.1

[PATCH v5 1/5] PCI: Add msi_controller setup_irqs() method for special multivector setup

[Lucas Stach]

Add a msi_controller setup_irqs() method so MSI chip providers can
implement their own multivector MSI setup.

[bhelgaas: changelog]
Signed-off-by: Lucas Stach <l.stach@pengutronix.de>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Reviewed-by: Pratyush Anand <pratyush.anand@gmail.com>
---
 drivers/pci/msi.c   | 3 +++
 include/linux/msi.h | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index d4497141d083..74319f497656 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -105,9 +105,12 @@ void __weak arch_teardown_msi_irq(unsigned int irq)
 
 int __weak arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
 {
+	struct msi_controller *chip = dev->bus->msi;
 	struct msi_desc *entry;
 	int ret;
 
+	if (chip && chip->setup_irqs)
+		return chip->setup_irqs(chip, dev, nvec, type);
 	/*
 	 * If an architecture wants to support multiple MSI, it needs to
 	 * override arch_setup_msi_irqs()
diff --git a/include/linux/msi.h b/include/linux/msi.h
index ad939d0ba816..0be5db110bfa 100644
--- a/include/linux/msi.h
+++ b/include/linux/msi.h
@@ -163,6 +163,8 @@ struct msi_controller {
 
 	int (*setup_irq)(struct msi_controller *chip, struct pci_dev *dev,
 			 struct msi_desc *desc);
+	int (*setup_irqs)(struct msi_controller *chip, struct pci_dev *dev,
+			  int nvec, int type);
 	void (*teardown_irq)(struct msi_controller *chip, unsigned int irq);
 };
 
-- 
2.5.1

[PATCH v5 2/5] PCI: designware: Factor out MSI msg setup

[Lucas Stach]

Factor out the PCI MSI message setup from the single MSI setup function.
This will be reused by the multivector MSI setup.

No functional change yet.

Signed-off-by: Lucas Stach <l.stach@pengutronix.de>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Acked-by: Pratyush Anand <pratyush.anand@gmail.com>
---
 drivers/pci/host/pcie-designware.c | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/drivers/pci/host/pcie-designware.c b/drivers/pci/host/pcie-designware.c
index 52aa6e34002b..7a168ea25e2f 100644
--- a/drivers/pci/host/pcie-designware.c
+++ b/drivers/pci/host/pcie-designware.c
@@ -293,19 +293,9 @@ no_valid_irq:
 	return -ENOSPC;
 }
 
-static int dw_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
-			struct msi_desc *desc)
+static void dw_msi_setup_msg(struct pcie_port *pp, unsigned int irq, u32 pos)
 {
-	int irq, pos;
 	struct msi_msg msg;
-	struct pcie_port *pp = sys_to_pcie(pdev->bus->sysdata);
-
-	if (desc->msi_attrib.is_msix)
-		return -EINVAL;
-
-	irq = assign_irq(1, desc, &pos);
-	if (irq < 0)
-		return irq;
 
 	if (pp->ops->get_msi_addr)
 		msg.address_lo = pp->ops->get_msi_addr(pp);
@@ -319,6 +309,22 @@ static int dw_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
 		msg.data = pos;
 
 	pci_write_msi_msg(irq, &msg);
+}
+
+static int dw_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
+			struct msi_desc *desc)
+{
+	int irq, pos;
+	struct pcie_port *pp = sys_to_pcie(pdev->bus->sysdata);
+
+	if (desc->msi_attrib.is_msix)
+		return -EINVAL;
+
+	irq = assign_irq(1, desc, &pos);
+	if (irq < 0)
+		return irq;
+
+	dw_msi_setup_msg(pp, irq, pos);
 
 	return 0;
 }
-- 
2.5.1

[PATCH v5 3/5] PCI: designware: Implement multivector MSI IRQ setup

[Lucas Stach]

Implement multivector MSI IRQ setup.  This allows to set up and use multiple
MSI IRQs per device.

[bhelgaas: changelog, use -EINVAL instead of -ENOSYS]
Signed-off-by: Lucas Stach <l.stach@pengutronix.de>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Acked-by: Pratyush Anand <pratyush.anand@gmail.com>
---
 drivers/pci/host/pcie-designware.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/drivers/pci/host/pcie-designware.c b/drivers/pci/host/pcie-designware.c
index 7a168ea25e2f..3a2b8bce27a6 100644
--- a/drivers/pci/host/pcie-designware.c
+++ b/drivers/pci/host/pcie-designware.c
@@ -286,6 +286,9 @@ static int assign_irq(int no_irqs, struct msi_desc *desc, int *pos)
 	}
 
 	*pos = pos0;
+	desc->nvec_used = no_irqs;
+	desc->msi_attrib.multiple = order_base_2(no_irqs);
+
 	return irq;
 
 no_valid_irq:
@@ -329,6 +332,33 @@ static int dw_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
 	return 0;
 }
 
+static int dw_msi_setup_irqs(struct msi_controller *chip, struct pci_dev *pdev,
+			     int nvec, int type)
+{
+#ifdef CONFIG_PCI_MSI
+	int irq, pos;
+	struct msi_desc *desc;
+	struct pcie_port *pp = sys_to_pcie(pdev->bus->sysdata);
+
+	/* MSI-X interrupts are not supported */
+	if (type == PCI_CAP_ID_MSIX)
+		return -EINVAL;
+
+	WARN_ON(!list_is_singular(&pdev->dev.msi_list));
+	desc = list_entry(pdev->dev.msi_list.next, struct msi_desc, list);
+
+	irq = assign_irq(nvec, desc, &pos);
+	if (irq < 0)
+		return irq;
+
+	dw_msi_setup_msg(pp, irq, pos);
+
+	return 0;
+#else
+	return -EINVAL;
+#endif
+}
+
 static void dw_msi_teardown_irq(struct msi_controller *chip, unsigned int irq)
 {
 	struct irq_data *data = irq_get_irq_data(irq);
@@ -340,6 +370,7 @@ static void dw_msi_teardown_irq(struct msi_controller *chip, unsigned int irq)
 
 static struct msi_controller dw_pcie_msi_chip = {
 	.setup_irq = dw_msi_setup_irq,
+	.setup_irqs = dw_msi_setup_irqs,
 	.teardown_irq = dw_msi_teardown_irq,
 };
 
-- 
2.5.1

[PATCH v5 4/5] PCI: designware: Make get_msi_addr() return phys_addr_t, not u32

[Lucas Stach]

Make get_msi_addr() return phys_addr_t, not u32.  This allows the MSI
target address to be above 4GB for 64bit or PAE systems.

No functional change for the current 32bit platform users as phys_addr_t
maps to u32 for them.

[bhelgaas: changelog]
Signed-off-by: Lucas Stach <l.stach@pengutronix.de>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Acked-by: Pratyush Anand <pratyush.anand@gmail.com>
---
 drivers/pci/host/pci-keystone-dw.c | 2 +-
 drivers/pci/host/pci-keystone.h    | 2 +-
 drivers/pci/host/pcie-designware.h | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/pci/host/pci-keystone-dw.c b/drivers/pci/host/pci-keystone-dw.c
index e71da991949b..3cf55cdb7f87 100644
--- a/drivers/pci/host/pci-keystone-dw.c
+++ b/drivers/pci/host/pci-keystone-dw.c
@@ -70,7 +70,7 @@ static inline void update_reg_offset_bit_pos(u32 offset, u32 *reg_offset,
 	*bit_pos = offset >> 3;
 }
 
-u32 ks_dw_pcie_get_msi_addr(struct pcie_port *pp)
+phys_addr_t ks_dw_pcie_get_msi_addr(struct pcie_port *pp)
 {
 	struct keystone_pcie *ks_pcie = to_keystone_pcie(pp);
 
diff --git a/drivers/pci/host/pci-keystone.h b/drivers/pci/host/pci-keystone.h
index 478d932b602d..f0944e8c4b02 100644
--- a/drivers/pci/host/pci-keystone.h
+++ b/drivers/pci/host/pci-keystone.h
@@ -37,7 +37,7 @@ struct keystone_pcie {
 
 /* Keystone DW specific MSI controller APIs/definitions */
 void ks_dw_pcie_handle_msi_irq(struct keystone_pcie *ks_pcie, int offset);
-u32 ks_dw_pcie_get_msi_addr(struct pcie_port *pp);
+phys_addr_t ks_dw_pcie_get_msi_addr(struct pcie_port *pp);
 
 /* Keystone specific PCI controller APIs */
 void ks_dw_pcie_enable_legacy_irqs(struct keystone_pcie *ks_pcie);
diff --git a/drivers/pci/host/pcie-designware.h b/drivers/pci/host/pcie-designware.h
index d0bbd276840d..35123d9362c5 100644
--- a/drivers/pci/host/pcie-designware.h
+++ b/drivers/pci/host/pcie-designware.h
@@ -70,7 +70,7 @@ struct pcie_host_ops {
 	void (*host_init)(struct pcie_port *pp);
 	void (*msi_set_irq)(struct pcie_port *pp, int irq);
 	void (*msi_clear_irq)(struct pcie_port *pp, int irq);
-	u32 (*get_msi_addr)(struct pcie_port *pp);
+	phys_addr_t (*get_msi_addr)(struct pcie_port *pp);
 	u32 (*get_msi_data)(struct pcie_port *pp, int pos);
 	void (*scan_bus)(struct pcie_port *pp);
 	int (*msi_host_init)(struct pcie_port *pp, struct msi_controller *chip);
-- 
2.5.1

[PATCH v5 5/5] PCI: designware: Set up high part of MSI target address

[Lucas Stach]

Set up the high part of the MSI target address to allow the MSI target to
be above 4GB on 64bit and PAE systems.

[bhelgaas: changelog]
Signed-off-by: Lucas Stach <l.stach@pengutronix.de>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Acked-by: Pratyush Anand <pratyush.anand@gmail.com>
---
 drivers/pci/host/pcie-designware.c | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/drivers/pci/host/pcie-designware.c b/drivers/pci/host/pcie-designware.c
index 3a2b8bce27a6..4eada6bc5fef 100644
--- a/drivers/pci/host/pcie-designware.c
+++ b/drivers/pci/host/pcie-designware.c
@@ -205,12 +205,16 @@ irqreturn_t dw_handle_msi_irq(struct pcie_port *pp)
 
 void dw_pcie_msi_init(struct pcie_port *pp)
 {
+	u64 msi_target;
+
 	pp->msi_data = __get_free_pages(GFP_KERNEL, 0);
+	msi_target = virt_to_phys((void *)pp->msi_data);
 
 	/* program the msi_data */
 	dw_pcie_wr_own_conf(pp, PCIE_MSI_ADDR_LO, 4,
-			virt_to_phys((void *)pp->msi_data));
-	dw_pcie_wr_own_conf(pp, PCIE_MSI_ADDR_HI, 4, 0);
+			    (u32)(msi_target & 0xffffffff));
+	dw_pcie_wr_own_conf(pp, PCIE_MSI_ADDR_HI, 4,
+			    (u32)(msi_target >> 32 & 0xffffffff));
 }
 
 static void dw_pcie_msi_clear_irq(struct pcie_port *pp, int irq)
@@ -299,12 +303,15 @@ no_valid_irq:
 static void dw_msi_setup_msg(struct pcie_port *pp, unsigned int irq, u32 pos)
 {
 	struct msi_msg msg;
+	u64 msi_target;
 
 	if (pp->ops->get_msi_addr)
-		msg.address_lo = pp->ops->get_msi_addr(pp);
+		msi_target = pp->ops->get_msi_addr(pp);
 	else
-		msg.address_lo = virt_to_phys((void *)pp->msi_data);
-	msg.address_hi = 0x0;
+		msi_target = virt_to_phys((void *)pp->msi_data);
+
+	msg.address_lo = (u32)(msi_target & 0xffffffff);
+	msg.address_hi = (u32)(msi_target >> 32 & 0xffffffff);
 
 	if (pp->ops->get_msi_data)
 		msg.data = pp->ops->get_msi_data(pp, pos);
-- 
2.5.1

Re: [PATCH v5 0/5] Designware host multivector MSI and 64bit MSI fixes

[Bjorn Helgaas]

On Fri, Sep 18, 2015 at 05:12:34PM +0200, Lucas Stach wrote:
> This is a reworked version of the multivector MSI support for the designware
> PCIe host controller driver.
> 
> v3 added patch 2 to share more code between the single and multivector MSI
> code paths.
> 
> While not really related to this topic patches 4+5 fix the MSI message setup
> to work on 64bit and 32bit PAE systems. I included them in this series as
> they depend on patch 2.
> 
> v4 fixes the last patch to also set up the high part of the MSI target
> address, when calling the common DW MSI init function.
> 
> v5 fixes up clashes with the PCI and MSI changes in v4.3-rc1 and fixes
> one build failure introduced in the last version.
> 
> Regards,
> Lucas
> 
> Lucas Stach (5):
>   PCI: Add msi_controller setup_irqs() method for special multivector
>     setup
>   PCI: designware: Factor out MSI msg setup
>   PCI: designware: Implement multivector MSI IRQ setup
>   PCI: designware: Make get_msi_addr() return phys_addr_t, not u32
>   PCI: designware: Set up high part of MSI target address
> 
>  drivers/pci/host/pci-keystone-dw.c |  2 +-
>  drivers/pci/host/pci-keystone.h    |  2 +-
>  drivers/pci/host/pcie-designware.c | 70 +++++++++++++++++++++++++++++++-------
>  drivers/pci/host/pcie-designware.h |  2 +-
>  drivers/pci/msi.c                  |  3 ++
>  include/linux/msi.h                |  2 ++
>  6 files changed, 65 insertions(+), 16 deletions(-)

Applied to pci/host-designware for v4.4, replacing the v4 patches, thanks!