Re: [PATCH v3 1/5] PCI: dwc: Convert msi_irq to the array

[Dmitry Baryshkov <dmitry.baryshkov@linaro.org>]

On 28/04/2022 09:06, Manivannan Sadhasivam wrote:
> On Wed, Apr 27, 2022 at 07:59:57PM +0300, Dmitry Baryshkov wrote:
>> On 27/04/2022 17:13, Manivannan Sadhasivam wrote:
>>> On Wed, Apr 27, 2022 at 03:16:49PM +0300, Dmitry Baryshkov wrote:
>>>> Qualcomm version of DWC PCIe controller supports more than 32 MSI
>>>> interrupts, but they are routed to separate interrupts in groups of 32
>>>> vectors. To support such configuration, change the msi_irq field into an
>>>> array. Let the DWC core handle all interrupts that were set in this
>>>> array.
>>>>
>>>
>>> Instead of defining it as an array, can we allocate it dynamically in the
>>> controller drivers instead? This has two benefits:
>>>
>>> 1. There is no need of using a dedicated flag.
>>> 2. Controller drivers that don't support MSIs can pass NULL and in the core we
>>> can use platform_get_irq_byname_optional() to get supported number of MSIs from
>>> devicetree.
>>
>> I think using dynamic allocation would make code worse. It would add
>> additional checks here and there.
>>
> 
> I take back my suggestion of allocating the memory for msi_irq in controller
> drivers. It should be done in the designware-host instead.
> 
> We already know how many MSIs are supported by the platform using num_vectors.
> So we should just allocate msi_irqs of num_vectors length in
> dw_pcie_host_init() and populate it using platform_get_irq_byname_optional().
> 
> I don't think this can make the code worse.
> 
>> If you don't like this design. I have an alternative suggestion: export the
>> dw_chained_msi_irq() and move allocation of all MSIs to the pcie-qcom code.
>> Would that be better? I'm not sure whether this multi-host-IRQ design is
>> used on other DWC platforms or not.
>>
> 
> No, I think the allocation should belong to designware-host.

Granted that at least tegra and iMX tie all MSI vectors to a single host 
interrupt, I think it's impractical to assume that other hosts would 
benefit from such allocation. Let's move support for split IRQs into the 
pcie-qcom.c. We can make this generic later if any other host would use 
a separate per-"endpoint" (per-group) IRQ.

> 
> Thanks,
> Mani
> 
>>>
>>> Thanks,
>>> Mani
>>>
>>>> Signed-off-by: Dmitry Baryshkov <dmitry.baryshkov@linaro.org>
>>>> ---
>>>>    drivers/pci/controller/dwc/pci-dra7xx.c       |  2 +-
>>>>    drivers/pci/controller/dwc/pci-exynos.c       |  2 +-
>>>>    .../pci/controller/dwc/pcie-designware-host.c | 30 +++++++++++--------
>>>>    drivers/pci/controller/dwc/pcie-designware.h  |  2 +-
>>>>    drivers/pci/controller/dwc/pcie-keembay.c     |  2 +-
>>>>    drivers/pci/controller/dwc/pcie-spear13xx.c   |  2 +-
>>>>    drivers/pci/controller/dwc/pcie-tegra194.c    |  2 +-
>>>>    7 files changed, 24 insertions(+), 18 deletions(-)
>>>>
>>>> diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
>>>> index dfcdeb432dc8..0919c96dcdbd 100644
>>>> --- a/drivers/pci/controller/dwc/pci-dra7xx.c
>>>> +++ b/drivers/pci/controller/dwc/pci-dra7xx.c
>>>> @@ -483,7 +483,7 @@ static int dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx,
>>>>    		return pp->irq;
>>>>    	/* MSI IRQ is muxed */
>>>> -	pp->msi_irq = -ENODEV;
>>>> +	pp->msi_irq[0] = -ENODEV;
>>>>    	ret = dra7xx_pcie_init_irq_domain(pp);
>>>>    	if (ret < 0)
>>>> diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c
>>>> index 467c8d1cd7e4..4f2010bd9cd7 100644
>>>> --- a/drivers/pci/controller/dwc/pci-exynos.c
>>>> +++ b/drivers/pci/controller/dwc/pci-exynos.c
>>>> @@ -292,7 +292,7 @@ static int exynos_add_pcie_port(struct exynos_pcie *ep,
>>>>    	}
>>>>    	pp->ops = &exynos_pcie_host_ops;
>>>> -	pp->msi_irq = -ENODEV;
>>>> +	pp->msi_irq[0] = -ENODEV;
>>>>    	ret = dw_pcie_host_init(pp);
>>>>    	if (ret) {
>>>> diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
>>>> index 2fa86f32d964..5d90009a0f73 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-designware-host.c
>>>> +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
>>>> @@ -257,8 +257,11 @@ int dw_pcie_allocate_domains(struct pcie_port *pp)
>>>>    static void dw_pcie_free_msi(struct pcie_port *pp)
>>>>    {
>>>> -	if (pp->msi_irq)
>>>> -		irq_set_chained_handler_and_data(pp->msi_irq, NULL, NULL);
>>>> +	u32 ctrl;
>>>> +
>>>> +	for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++)
>>>> +		if (pp->msi_irq[ctrl])
>>>> +			irq_set_chained_handler_and_data(pp->msi_irq[ctrl], NULL, NULL);
>>>>    	irq_domain_remove(pp->msi_domain);
>>>>    	irq_domain_remove(pp->irq_domain);
>>>> @@ -368,13 +371,15 @@ int dw_pcie_host_init(struct pcie_port *pp)
>>>>    			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
>>>>    				pp->irq_mask[ctrl] = ~0;
>>>> -			if (!pp->msi_irq) {
>>>> -				pp->msi_irq = platform_get_irq_byname_optional(pdev, "msi");
>>>> -				if (pp->msi_irq < 0) {
>>>> -					pp->msi_irq = platform_get_irq(pdev, 0);
>>>> -					if (pp->msi_irq < 0)
>>>> -						return pp->msi_irq;
>>>> +			if (!pp->msi_irq[0]) {
>>>> +				int irq = platform_get_irq_byname_optional(pdev, "msi");
>>>> +
>>>> +				if (irq < 0) {
>>>> +					irq = platform_get_irq(pdev, 0);
>>>> +					if (irq < 0)
>>>> +						return irq;
>>>>    				}
>>>> +				pp->msi_irq[0] = irq;
>>>>    			}
>>>>    			pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
>>>> @@ -383,10 +388,11 @@ int dw_pcie_host_init(struct pcie_port *pp)
>>>>    			if (ret)
>>>>    				return ret;
>>>> -			if (pp->msi_irq > 0)
>>>> -				irq_set_chained_handler_and_data(pp->msi_irq,
>>>> -							    dw_chained_msi_isr,
>>>> -							    pp);
>>>> +			for (ctrl = 0; ctrl < num_ctrls; ctrl++)
>>>> +				if (pp->msi_irq[ctrl] > 0)
>>>> +					irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
>>>> +									 dw_chained_msi_isr,
>>>> +									 pp);
>>>>    			ret = dma_set_mask(pci->dev, DMA_BIT_MASK(32));
>>>>    			if (ret)
>>>> diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
>>>> index 7d6e9b7576be..9c1a38b0a6b3 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-designware.h
>>>> +++ b/drivers/pci/controller/dwc/pcie-designware.h
>>>> @@ -187,7 +187,7 @@ struct pcie_port {
>>>>    	u32			io_size;
>>>>    	int			irq;
>>>>    	const struct dw_pcie_host_ops *ops;
>>>> -	int			msi_irq;
>>>> +	int			msi_irq[MAX_MSI_CTRLS];
>>>>    	struct irq_domain	*irq_domain;
>>>>    	struct irq_domain	*msi_domain;
>>>>    	u16			msi_msg;
>>>> diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c
>>>> index 1ac29a6eef22..297e6e926c00 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-keembay.c
>>>> +++ b/drivers/pci/controller/dwc/pcie-keembay.c
>>>> @@ -338,7 +338,7 @@ static int keembay_pcie_add_pcie_port(struct keembay_pcie *pcie,
>>>>    	int ret;
>>>>    	pp->ops = &keembay_pcie_host_ops;
>>>> -	pp->msi_irq = -ENODEV;
>>>> +	pp->msi_irq[0] = -ENODEV;
>>>>    	ret = keembay_pcie_setup_msi_irq(pcie);
>>>>    	if (ret)
>>>> diff --git a/drivers/pci/controller/dwc/pcie-spear13xx.c b/drivers/pci/controller/dwc/pcie-spear13xx.c
>>>> index 1569e82b5568..cc7776833810 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-spear13xx.c
>>>> +++ b/drivers/pci/controller/dwc/pcie-spear13xx.c
>>>> @@ -172,7 +172,7 @@ static int spear13xx_add_pcie_port(struct spear13xx_pcie *spear13xx_pcie,
>>>>    	}
>>>>    	pp->ops = &spear13xx_pcie_host_ops;
>>>> -	pp->msi_irq = -ENODEV;
>>>> +	pp->msi_irq[0] = -ENODEV;
>>>>    	ret = dw_pcie_host_init(pp);
>>>>    	if (ret) {
>>>> diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
>>>> index b1b5f836a806..e75712db85b0 100644
>>>> --- a/drivers/pci/controller/dwc/pcie-tegra194.c
>>>> +++ b/drivers/pci/controller/dwc/pcie-tegra194.c
>>>> @@ -2271,7 +2271,7 @@ static void tegra194_pcie_shutdown(struct platform_device *pdev)
>>>>    	disable_irq(pcie->pci.pp.irq);
>>>>    	if (IS_ENABLED(CONFIG_PCI_MSI))
>>>> -		disable_irq(pcie->pci.pp.msi_irq);
>>>> +		disable_irq(pcie->pci.pp.msi_irq[0]);
>>>>    	tegra194_pcie_pme_turnoff(pcie);
>>>>    	tegra_pcie_unconfig_controller(pcie);
>>>> -- 
>>>> 2.35.1
>>>>
>>
>>
>> -- 
>> With best wishes
>> Dmitry


-- 
With best wishes
Dmitry