Re: [PATCH v8 3/7] PCI: qcom: Add ICC bandwidth vote for CPU to PCIe path

[Krishna Chaitanya Chundru <quic_krichai@quicinc.com>]

On 3/4/2024 11:11 PM, Manivannan Sadhasivam wrote:
> On Sat, Mar 02, 2024 at 09:29:57AM +0530, Krishna chaitanya chundru wrote:
>> To access PCIe registers, PCIe BAR space, config space the CPU-PCIe
>> ICC (interconnect consumers) path should be voted otherwise it may
>> lead to NoC (Network on chip) timeout. We are surviving because of
>> other driver vote for this path.
>>
>> As there is less access on this path compared to PCIe to mem path
>> add minimum vote i.e 1KBps bandwidth always.
> 
> Please add the info that 1KBps is what shared by the HW team.
> 
Ack to all the comments
>>
>> When suspending, disable this path after register space access
>> is done.
>>
>> Reviewed-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org>
>> Signed-off-by: Krishna chaitanya chundru <quic_krichai@quicinc.com>
>> ---
>>   drivers/pci/controller/dwc/pcie-qcom.c | 38 ++++++++++++++++++++++++++++++++--
>>   1 file changed, 36 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
>> index 10f2d0bb86be..a0266bfe71f1 100644
>> --- a/drivers/pci/controller/dwc/pcie-qcom.c
>> +++ b/drivers/pci/controller/dwc/pcie-qcom.c
>> @@ -240,6 +240,7 @@ struct qcom_pcie {
>>   	struct phy *phy;
>>   	struct gpio_desc *reset;
>>   	struct icc_path *icc_mem;
>> +	struct icc_path *icc_cpu;
>>   	const struct qcom_pcie_cfg *cfg;
>>   	struct dentry *debugfs;
>>   	bool suspended;
>> @@ -1372,6 +1373,9 @@ static int qcom_pcie_icc_init(struct qcom_pcie *pcie)
>>   	if (IS_ERR(pcie->icc_mem))
>>   		return PTR_ERR(pcie->icc_mem);
>>   
>> +	pcie->icc_cpu = devm_of_icc_get(pci->dev, "cpu-pcie");
>> +	if (IS_ERR(pcie->icc_cpu))
>> +		return PTR_ERR(pcie->icc_cpu);
>>   	/*
>>   	 * Some Qualcomm platforms require interconnect bandwidth constraints
>>   	 * to be set before enabling interconnect clocks.
>> @@ -1381,7 +1385,19 @@ static int qcom_pcie_icc_init(struct qcom_pcie *pcie)
>>   	 */
>>   	ret = icc_set_bw(pcie->icc_mem, 0, QCOM_PCIE_LINK_SPEED_TO_BW(1));
>>   	if (ret) {
>> -		dev_err(pci->dev, "failed to set interconnect bandwidth: %d\n",
>> +		dev_err(pci->dev, "failed to set interconnect bandwidth for pcie-mem: %d\n",
> 
> "PCIe-MEM"
> 
>> +			ret);
>> +		return ret;
>> +	}
>> +
>> +	/*
>> +	 * The config space, BAR space and registers goes through cpu-pcie path
>> +	 * Set peak bandwidth to 1KBps as recommended by HW team for this path
>> +	 * all the time.
> 
> How about,
> 
> 	"Since the CPU-PCIe path is only used for activities like register
> 	access, Config/BAR space access, HW team has recommended to use a
> 	minimal bandwidth of 1KBps just to keep the link active."
> 
>> +	 */
>> +	ret = icc_set_bw(pcie->icc_cpu, 0, kBps_to_icc(1));
>> +	if (ret) {
>> +		dev_err(pci->dev, "failed to set interconnect bandwidth for cpu-pcie: %d\n",
>>   			ret);
>>   		return ret;
>>   	}
>> @@ -1573,7 +1589,7 @@ static int qcom_pcie_suspend_noirq(struct device *dev)
>>   	 */
>>   	ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1));
>>   	if (ret) {
>> -		dev_err(dev, "Failed to set interconnect bandwidth: %d\n", ret);
>> +		dev_err(dev, "Failed to set interconnect bandwidth for pcie-mem: %d\n", ret);
> 
> "PCIe-MEM"
> 
>>   		return ret;
>>   	}
>>   
>> @@ -1597,6 +1613,18 @@ static int qcom_pcie_suspend_noirq(struct device *dev)
>>   		pcie->suspended = true;
>>   	}
>>   
>> +	/* Remove CPU path vote after all the register access is done */
> 
> "Remove the vote for CPU-PCIe path now, since at this point onwards, no register
> access will be done."
> 
>> +	ret = icc_disable(pcie->icc_cpu);
>> +	if (ret) {
>> +		dev_err(dev, "failed to disable icc path of cpu-pcie: %d\n", ret);
> 
> "CPU-PCIe"
> 
>> +		if (pcie->suspended) {
>> +			qcom_pcie_host_init(&pcie->pci->pp);
> 
> Interesting. So if icc_disable() fails, can the IP continue to function?
>
As the ICC already enable before icc_disable() fails, the IP should work.
  - Krishna Chaitanya.
>> +			pcie->suspended = false;
>> +		}
>> +		qcom_pcie_icc_update(pcie);
>> +		return ret;
>> +	}
>> +
>>   	return 0;
>>   }
>>   
>> @@ -1605,6 +1633,12 @@ static int qcom_pcie_resume_noirq(struct device *dev)
>>   	struct qcom_pcie *pcie = dev_get_drvdata(dev);
>>   	int ret;
>>   
>> +	ret = icc_enable(pcie->icc_cpu);
>> +	if (ret) {
>> +		dev_err(dev, "failed to enable icc path of cpu-pcie: %d\n", ret);
> 
> "CPU-PCIe"
> 
> - Mani
>