Re: [PATCH v4 4/7] PCI: dwc: endpoint: Allow EPF drivers to configure the size of Resizable BARs

[Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>]

On Fri, Jan 31, 2025 at 07:29:53PM +0100, Niklas Cassel wrote:
> The DWC databook specifies three different BARn_SIZING_SCHEME_N
> - Fixed Mask (0)
> - Programmable Mask (1)
> - Resizable BAR (2)
> 
> Each of these sizing schemes have different instructions for how to
> initialize the BAR.
> 
> The DWC driver currently does not support resizable BARs.
> 
> Instead, in order to somewhat support resizable BARs, the DWC EP driver
> currently has an ugly hack that force sets a resizable BAR to 1 MB, if
> such a BAR is detected.
> 
> Additionally, this hack only works if the DWC glue driver also has lied
> in their EPC features, and claimed that the resizable BAR is a 1 MB fixed
> size BAR.
> 
> This is unintuitive (as you somehow need to know that you need to lie in
> your EPC features), but other than that it is overly restrictive, since a
> resizable BAR is capable of supporting sizes different than 1 MB.
> 
> Add proper support for resizable BARs in the DWC EP driver.
> 
> Note that the pci_epc_set_bar() API takes a struct pci_epf_bar which tells
> the EPC driver how it wants to configure the BAR.
> 
> struct pci_epf_bar only has a single size struct member.
> 
> This means that an EPC driver will only be able to set a single supported
> size. This is perfectly fine, as we do not need the complexity of allowing
> a host to change the size of the BAR. If someone ever wants to support
> resizing a resizable BAR, the pci_epc_set_bar() API can be extended in the
> future.
> 
> With these changes, we allow an EPF driver to configure the size of
> Resizable BARs, rather than forcing them to a 1 MB size.
> 
> This means that an EPC driver does not need to lie in EPC features, and an
> EPF driver will be able to set an arbitrary size (not be forced to a 1 MB
> size), just like BAR_PROGRAMMABLE.
> 
> Signed-off-by: Niklas Cassel <cassel@kernel.org>

Reviewed-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>

- Mani

> ---
>  .../pci/controller/dwc/pcie-designware-ep.c   | 182 ++++++++++++++++--
>  1 file changed, 167 insertions(+), 15 deletions(-)
> 
> diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
> index 6b494781da42..72418160e658 100644
> --- a/drivers/pci/controller/dwc/pcie-designware-ep.c
> +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
> @@ -223,6 +223,125 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
>  	ep->bar_to_atu[bar] = 0;
>  }
>  
> +static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie *pci,
> +						enum pci_barno bar)
> +{
> +	u32 reg, bar_index;
> +	unsigned int offset, nbars;
> +	int i;
> +
> +	offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR);
> +	if (!offset)
> +		return offset;
> +
> +	reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL);
> +	nbars = (reg & PCI_REBAR_CTRL_NBAR_MASK) >> PCI_REBAR_CTRL_NBAR_SHIFT;
> +
> +	for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) {
> +		reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL);
> +		bar_index = reg & PCI_REBAR_CTRL_BAR_IDX;
> +		if (bar_index == bar)
> +			return offset;
> +	}
> +
> +	return 0;
> +}
> +
> +static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no,
> +					struct pci_epf_bar *epf_bar)
> +{
> +	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
> +	enum pci_barno bar = epf_bar->barno;
> +	size_t size = epf_bar->size;
> +	int flags = epf_bar->flags;
> +	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
> +	unsigned int rebar_offset;
> +	u32 rebar_cap, rebar_ctrl;
> +	int ret;
> +
> +	rebar_offset = dw_pcie_ep_get_rebar_offset(pci, bar);
> +	if (!rebar_offset)
> +		return -EINVAL;
> +
> +	ret = pci_epc_bar_size_to_rebar_cap(size, &rebar_cap);
> +	if (ret)
> +		return ret;
> +
> +	dw_pcie_dbi_ro_wr_en(pci);
> +
> +	/*
> +	 * A BAR mask should not be written for a resizable BAR. The BAR mask
> +	 * is automatically derived by the controller every time the "selected
> +	 * size" bits are updated, see "Figure 3-26 Resizable BAR Example for
> +	 * 32-bit Memory BAR0" in DWC EP databook 5.96a. We simply need to write
> +	 * BIT(0) to set the BAR enable bit.
> +	 */
> +	dw_pcie_ep_writel_dbi2(ep, func_no, reg, BIT(0));
> +	dw_pcie_ep_writel_dbi(ep, func_no, reg, flags);
> +
> +	if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
> +		dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, 0);
> +		dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0);
> +	}
> +
> +	/*
> +	 * Bits 31:0 in PCI_REBAR_CAP define "supported sizes" bits for sizes
> +	 * 1 MB to 128 TB. Bits 31:16 in PCI_REBAR_CTRL define "supported sizes"
> +	 * bits for sizes 256 TB to 8 EB. Disallow sizes 256 TB to 8 EB.
> +	 */
> +	rebar_ctrl = dw_pcie_readl_dbi(pci, rebar_offset + PCI_REBAR_CTRL);
> +	rebar_ctrl &= ~GENMASK(31, 16);
> +	dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl);
> +
> +	/*
> +	 * The "selected size" (bits 13:8) in PCI_REBAR_CTRL are automatically
> +	 * updated when writing PCI_REBAR_CAP, see "Figure 3-26 Resizable BAR
> +	 * Example for 32-bit Memory BAR0" in DWC EP databook 5.96a.
> +	 */
> +	dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CAP, rebar_cap);
> +
> +	dw_pcie_dbi_ro_wr_dis(pci);
> +
> +	return 0;
> +}
> +
> +static int dw_pcie_ep_set_bar_programmable(struct dw_pcie_ep *ep, u8 func_no,
> +					   struct pci_epf_bar *epf_bar)
> +{
> +	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
> +	enum pci_barno bar = epf_bar->barno;
> +	size_t size = epf_bar->size;
> +	int flags = epf_bar->flags;
> +	u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
> +
> +	dw_pcie_dbi_ro_wr_en(pci);
> +
> +	dw_pcie_ep_writel_dbi2(ep, func_no, reg, lower_32_bits(size - 1));
> +	dw_pcie_ep_writel_dbi(ep, func_no, reg, flags);
> +
> +	if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
> +		dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, upper_32_bits(size - 1));
> +		dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0);
> +	}
> +
> +	dw_pcie_dbi_ro_wr_dis(pci);
> +
> +	return 0;
> +}
> +
> +static enum pci_epc_bar_type dw_pcie_ep_get_bar_type(struct dw_pcie_ep *ep,
> +						     enum pci_barno bar)
> +{
> +	const struct pci_epc_features *epc_features;
> +
> +	if (!ep->ops->get_features)
> +		return BAR_PROGRAMMABLE;
> +
> +	epc_features = ep->ops->get_features(ep);
> +
> +	return epc_features->bar[bar].type;
> +}
> +
>  static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
>  			      struct pci_epf_bar *epf_bar)
>  {
> @@ -230,9 +349,9 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
>  	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
>  	enum pci_barno bar = epf_bar->barno;
>  	size_t size = epf_bar->size;
> +	enum pci_epc_bar_type bar_type;
>  	int flags = epf_bar->flags;
>  	int ret, type;
> -	u32 reg;
>  
>  	/*
>  	 * DWC does not allow BAR pairs to overlap, e.g. you cannot combine BARs
> @@ -264,19 +383,30 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
>  		goto config_atu;
>  	}
>  
> -	reg = PCI_BASE_ADDRESS_0 + (4 * bar);
> -
> -	dw_pcie_dbi_ro_wr_en(pci);
> -
> -	dw_pcie_ep_writel_dbi2(ep, func_no, reg, lower_32_bits(size - 1));
> -	dw_pcie_ep_writel_dbi(ep, func_no, reg, flags);
> -
> -	if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
> -		dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, upper_32_bits(size - 1));
> -		dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0);
> +	bar_type = dw_pcie_ep_get_bar_type(ep, bar);
> +	switch (bar_type) {
> +	case BAR_FIXED:
> +		/*
> +		 * There is no need to write a BAR mask for a fixed BAR (except
> +		 * to write 1 to the LSB of the BAR mask register, to enable the
> +		 * BAR). Write the BAR mask regardless. (The fixed bits in the
> +		 * BAR mask register will be read-only anyway.)
> +		 */
> +		fallthrough;
> +	case BAR_PROGRAMMABLE:
> +		ret = dw_pcie_ep_set_bar_programmable(ep, func_no, epf_bar);
> +		break;
> +	case BAR_RESIZABLE:
> +		ret = dw_pcie_ep_set_bar_resizable(ep, func_no, epf_bar);
> +		break;
> +	default:
> +		ret = -EINVAL;
> +		dev_err(pci->dev, "Invalid BAR type\n");
> +		break;
>  	}
>  
> -	dw_pcie_dbi_ro_wr_dis(pci);
> +	if (ret)
> +		return ret;
>  
>  config_atu:
>  	if (!(flags & PCI_BASE_ADDRESS_SPACE))
> @@ -710,9 +840,11 @@ EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit);
>  
>  static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci)
>  {
> +	struct dw_pcie_ep *ep = &pci->ep;
>  	unsigned int offset;
>  	unsigned int nbars;
> -	u32 reg, i;
> +	enum pci_barno bar;
> +	u32 reg, i, val;
>  
>  	offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR);
>  
> @@ -727,9 +859,29 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci)
>  		 * PCIe r6.0, sec 7.8.6.2 require us to support at least one
>  		 * size in the range from 1 MB to 512 GB. Advertise support
>  		 * for 1 MB BAR size only.
> +		 *
> +		 * For a BAR that has been configured via dw_pcie_ep_set_bar(),
> +		 * advertise support for only that size instead.
>  		 */
> -		for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL)
> -			dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, BIT(4));
> +		for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) {
> +			/*
> +			 * While the RESBAR_CAP_REG_* fields are sticky, the
> +			 * RESBAR_CTRL_REG_BAR_SIZE field is non-sticky (it is
> +			 * sticky in certain versions of DWC PCIe, but not all).
> +			 *
> +			 * RESBAR_CTRL_REG_BAR_SIZE is updated automatically by
> +			 * the controller when RESBAR_CAP_REG is written, which
> +			 * is why RESBAR_CAP_REG is written here.
> +			 */
> +			val = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL);
> +			bar = val & PCI_REBAR_CTRL_BAR_IDX;
> +			if (ep->epf_bar[bar])
> +				pci_epc_bar_size_to_rebar_cap(ep->epf_bar[bar]->size, &val);
> +			else
> +				val = BIT(4);
> +
> +			dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, val);
> +		}
>  	}
>  
>  	dw_pcie_setup(pci);
> -- 
> 2.48.1
> 

-- 
மணிவண்ணன் சதாசிவம்