Re: [PATCH v9 06/10] PCI: endpoint: pci-epf-vntb: Allow DMA and MW to share a BAR

[Frank Li <Frank.li@oss.nxp.com>]

On Tue, Nov 10, 2026 at 05:24:55PM +0900, Koichiro Den wrote:
> Some endpoint configurations have no spare BAR for DMA resources. Allow
> dma_bar to select an MW BAR and place the DMA ranges after that MW.
>
> DMA submaps have fixed BAR offsets, so a shared MW translation must
> cover its configured size. Keep the power-of-two restriction used for
> ordinary MWs. The size must also meet the EPC's alignment requirement
> for the following DMA submaps.
>
> Older hosts treat the whole BAR as an MW. Fall back to that layout when
> LINK_UP does not advertise MW/DMA boundary support, so that NTB keeps
> working without DMA: the enumerated BAR size is kept, the MW spans the
> whole BAR and the DMA submaps are removed.
>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>

Reviewed-by: Frank Li <Frank.Li@nxp.com>

> ---
> Changes in v9:
>   - Report BAR-sized address alignment for shared-BAR fallback.
>     (Sashiko, #1)
>   - Reject partial shared MW requests without letting ntb_transport
>     round them up. (Sashiko, #2)
>   - Check the shared BAR number and MW offset alignment at bind time.
>     (Sashiko)
>   - Drop R-b tag due to the changes.
>   - Link to v8: https://lore.kernel.org/r/20261022082230.3164834-7-den@valinux.co.jp/
>
> To reviewers (esp. Frank):
>   Compared to v8, this may look like replacing the size_align change
>   with an addr_align change, but they apply to different paths.
>   Let me explain for clarity:
>
>   M = configured MW size, B = full shared BAR size. Values are v8 -> v9.
>
>   Path                   size_align    addr_align
>   ---------------------  ------------  -------------------------
>   Non-shared MW          1 -> 1        4 KiB -> 4 KiB
>   Shared, DMA submaps    M -> 1        4 KiB -> 4 KiB
>   Shared, fallback       1 -> 1        4 KiB -> max(4 KiB, B)
>
>   With DMA submaps, size_align = M could make ntb_transport round a
>   partial request up to M while allocating a smaller buffer. Returning 1
>   lets the existing full-size check in mw_set_trans() reject that
>   request instead. Supporting smaller MWs is left for later. I doubt it
>   would be worth complicating, just to support max_mw_size limitation
>   for such shared BAR cases, which people may not use. Addresses
>   (Sashiko, #2).
>
>   In fallback mode, the mapping covers the whole BAR even for a smaller
>   request. DWC requires a B-aligned target, so report that alignment.
>   size_align stays 1, so smaller translations remain allowed. Addresses
>   (Sashiko, #1).
>
>  drivers/pci/endpoint/functions/pci-epf-vntb.c | 220 ++++++++++++++----
>  include/linux/pci-epf-ntb.h                   |   3 +
>  2 files changed, 178 insertions(+), 45 deletions(-)
>
> diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
> index 875a7a9e487f..ba2df2ee991c 100644
> --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
> +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
> @@ -84,15 +84,17 @@ enum epf_irq_slot {
>  #define MIN_DB_COUNT			(EPF_IRQ_DB_START + 1)
>
>  struct epf_ntb_dma {
> +	struct mutex lock; /* Serialize DMA submap and MW updates */
>  	struct epf_ntb_dma_ctrl ctrl;
>  	struct dma_chan *dchan[EDMA_MAX_RD_CH];
>  	void *bar_scratch;
>  	dma_addr_t bar_scratch_phys;
>  	size_t bar_scratch_size;
> -	struct pci_epf_bar_submap submap[EDMA_MAX_RD_CH + 2];
> +	struct pci_epf_bar_submap submap[EDMA_MAX_RD_CH + 3];
>  	struct pci_epf_bar_submap *reg_submap;
>  	unsigned int num_submap;
>  	u16 rd_ch_cnt;
> +	bool mw_fallback; /* The MW spans the whole shared BAR */
>  };
>
>  struct epf_ntb {
> @@ -229,6 +231,22 @@ static bool epf_ntb_is_bar_used(struct epf_ntb *ntb,
>  	return false;
>  }
>
> +static u64 epf_ntb_dma_bar_offset(struct epf_ntb *ntb, enum pci_barno barno)
> +{
> +	unsigned int i;
> +
> +	for (i = 0; i < ntb->num_mws; i++)
> +		if (ntb->epf_ntb_bar[BAR_MW1 + i] == barno)
> +			return ntb->mws_size[i];
> +
> +	return 0;
> +}
> +
> +static bool epf_ntb_dma_shares_bar(struct epf_ntb *ntb, enum pci_barno barno)
> +{
> +	return ntb->dma && ntb->epf_ntb_bar[BAR_DMA] == barno;
> +}
> +
>  struct epf_ntb_dma_filter {
>  	struct device *dev;
>  	int chan_id;
> @@ -353,6 +371,7 @@ static int epf_ntb_dw_edma_collect(struct epf_ntb *ntb,
>  	unsigned int i;
>  	size_t align;
>  	u32 next = 0;
> +	u64 offset;
>  	int ret;
>
>  	if (ctrl->u.dma_ctrl.reg_layout_data != EDMA_MF_EDMA_UNROLL)
> @@ -371,10 +390,30 @@ static int epf_ntb_dw_edma_collect(struct epf_ntb *ntb,
>  	if (!is_power_of_2(align))
>  		return -EINVAL;
>
> -	if (epf_ntb_is_bar_used(ntb, barno) ||
> -	    pci_epc_get_next_free_bar(features, barno) != barno)
> +	if (barno == ntb->epf_ntb_bar[BAR_CONFIG] ||
> +	    barno == ntb->epf_ntb_bar[BAR_DB])
>  		return -EINVAL;
>
> +	offset = epf_ntb_dma_bar_offset(ntb, barno);
> +	if (pci_epc_get_next_free_bar(features, barno) != barno ||
> +	    (!offset && epf_ntb_is_bar_used(ntb, barno)))
> +		return -EINVAL;
> +	if (offset > U32_MAX)
> +		return -EOVERFLOW;
> +	if (offset && (!is_power_of_2(offset) || !IS_ALIGNED(offset, align)))
> +		return -EINVAL;
> +	dma->ctrl.submap.offset = offset;
> +	next = offset;
> +	if (next) {
> +		/*
> +		 * submap[0] covers the MW prefix. Keep its temporary
> +		 * target at address 0 until the MW translation is
> +		 * installed.
> +		 */
> +		dma->submap[0].size = next;
> +		dma->num_submap = 1;
> +	}
> +
>  	dma->ctrl.magic = EPF_NTB_DMA_MAGIC;
>  	dma->ctrl.revision = EPF_NTB_DMA_REVISION;
>  	dma->ctrl.type = EPF_NTB_DMA_TYPE_DW_EDMA;
> @@ -408,7 +447,7 @@ static int epf_ntb_dw_edma_collect(struct epf_ntb *ntb,
>  		chan->desc_addr_lo = lower_32_bits(dma_addr);
>  		chan->desc_addr_hi = upper_32_bits(dma_addr);
>  	}
> -	dma->ctrl.submap.size = next;
> +	dma->ctrl.submap.size = next - dma->ctrl.submap.offset;
>
>  	dma_dev = ntb->epf->epc->dev.parent;
>  	for (i = 0; i < dma->rd_ch_cnt; i++) {
> @@ -481,6 +520,8 @@ static int epf_ntb_dma_collect(struct epf_ntb *ntb)
>  	if (!dma)
>  		return -ENOMEM;
>
> +	mutex_init(&dma->lock);
> +
>  	switch (ctrl->u.dma_ctrl.reg_layout) {
>  	case PCI_EPC_AUX_DMA_REG_LAYOUT_DW_EDMA:
>  		ret = epf_ntb_dw_edma_collect(ntb, dma, ctrl, resources, count);
> @@ -551,7 +592,7 @@ static int epf_ntb_dma_bar_init(struct epf_ntb *ntb)
>  		return -EOPNOTSUPP;
>
>  	barno = ntb->epf_ntb_bar[BAR_DMA];
> -	mapped_size = dma->ctrl.submap.size;
> +	mapped_size = dma->ctrl.submap.offset + dma->ctrl.submap.size;
>  	/*
>  	 * Submaps cannot be installed until the host assigns the BAR address.
>  	 * Use address 0 for the temporary BAR Match Mode mapping, as is done
> @@ -607,23 +648,29 @@ static void epf_ntb_dma_bar_clear(struct epf_ntb *ntb)
>  	memset(bar, 0, sizeof(*bar));
>  }
>
> -static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
> +static int epf_ntb_dma_set_bar_locked(struct epf_ntb *ntb, bool use_submap,
> +				      const dma_addr_t *mw_addr)
>  {
> -	struct pci_epf_bar_submap *old_submap;
>  	struct epf_ntb_dma *dma = ntb->dma;
> -	unsigned int old_num_submap;
> -	struct pci_epf_bar *bar;
> +	struct pci_epf_bar *bar, old_bar;
> +	dma_addr_t old_mw_addr;
>  	int restore, ret;
>
> -	if (!dma)
> -		return -EOPNOTSUPP;
> +	lockdep_assert_held(&dma->lock);
>
>  	bar = &ntb->epf->bar[ntb->epf_ntb_bar[BAR_DMA]];
> -
> -	old_submap = bar->submap;
> -	old_num_submap = bar->num_submap;
> -	bar->submap = dma->submap;
> -	bar->num_submap = dma->num_submap;
> +	old_bar = *bar;
> +	old_mw_addr = dma->submap[0].phys_addr;
> +	if (use_submap) {
> +		if (mw_addr)
> +			dma->submap[0].phys_addr = *mw_addr;
> +		bar->submap = dma->submap;
> +		bar->num_submap = dma->num_submap;
> +	} else {
> +		bar->submap = NULL;
> +		bar->num_submap = 0;
> +		bar->phys_addr = mw_addr ? *mw_addr : 0;
> +	}
>
>  	ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no,
>  			      ntb->epf->vfunc_no, bar);
> @@ -631,8 +678,8 @@ static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
>  		return 0;
>
>  	/* A failed dynamic update may have already removed the old mapping. */
> -	bar->submap = old_submap;
> -	bar->num_submap = old_num_submap;
> +	dma->submap[0].phys_addr = old_mw_addr;
> +	*bar = old_bar;
>  	restore = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no,
>  				  ntb->epf->vfunc_no, bar);
>  	if (restore)
> @@ -642,6 +689,51 @@ static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
>  	return ret;
>  }
>
> +static int epf_ntb_dma_set_bar(struct epf_ntb *ntb)
> +{
> +	struct epf_ntb_dma *dma = ntb->dma;
> +
> +	if (!dma)
> +		return -EOPNOTSUPP;
> +
> +	guard(mutex)(&dma->lock);
> +
> +	return epf_ntb_dma_set_bar_locked(ntb, true, NULL);
> +}
> +
> +static int epf_ntb_dma_link_up(struct epf_ntb *ntb, u32 argument)
> +{
> +	struct epf_ntb_dma *dma = ntb->dma;
> +	struct pci_epf_bar *bar;
> +	bool fallback;
> +	int ret;
> +
> +	if (!dma || !dma->ctrl.submap.offset)
> +		return 0;
> +
> +	fallback = !(argument & EPF_NTB_LINK_UP_DMA_MW_SUPPORTED);
> +
> +	guard(mutex)(&dma->lock);
> +
> +	if (fallback == dma->mw_fallback)
> +		return 0;
> +
> +	if (fallback) {
> +		dev_warn(&ntb->epf->dev,
> +			 "Host does not support MW/DMA BAR sharing, exposing the whole BAR as an MW\n");
> +		/* Replace installed DMA submaps until the MW is set up. */
> +		bar = &ntb->epf->bar[ntb->epf_ntb_bar[BAR_DMA]];
> +		if (bar->num_submap) {
> +			ret = epf_ntb_dma_set_bar_locked(ntb, false, NULL);
> +			if (ret)
> +				return ret;
> +		}
> +	}
> +	dma->mw_fallback = fallback;
> +
> +	return 0;
> +}
> +
>  /**
>   * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST
>   *   to access the memory window of HOST
> @@ -780,7 +872,9 @@ static void epf_ntb_cmd_handler(struct work_struct *work)
>  					     EPF_NTB_COMMAND_STATUS_OK;
>  		break;
>  	case EPF_NTB_COMMAND_LINK_UP:
> -		ret = epf_ntb_set_link_enabled(ntb, true);
> +		ret = epf_ntb_dma_link_up(ntb, argument);
> +		if (!ret)
> +			ret = epf_ntb_set_link_enabled(ntb, true);
>  		if (ret < 0)
>  			ctrl->command_status = EPF_NTB_COMMAND_STATUS_ERROR;
>  		else
> @@ -1269,6 +1363,7 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
>   */
>  static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
>  {
> +	bool shared;
>  	int ret = 0;
>  	int i;
>  	u64 size;
> @@ -1278,22 +1373,25 @@ static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
>  	for (i = 0; i < ntb->num_mws; i++) {
>  		size = ntb->mws_size[i];
>  		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
> -
> -		ntb->epf->bar[barno].barno = barno;
> -		ntb->epf->bar[barno].size = size;
> -		ntb->epf->bar[barno].addr = NULL;
> -		ntb->epf->bar[barno].phys_addr = 0;
> -		ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
> -				PCI_BASE_ADDRESS_MEM_TYPE_64 :
> -				PCI_BASE_ADDRESS_MEM_TYPE_32;
> -
> -		ret = pci_epc_set_bar(ntb->epf->epc,
> -				      ntb->epf->func_no,
> -				      ntb->epf->vfunc_no,
> -				      &ntb->epf->bar[barno]);
> -		if (ret) {
> -			dev_err(dev, "MW set failed\n");
> -			goto err_alloc_mem;
> +		shared = epf_ntb_dma_shares_bar(ntb, barno);
> +
> +		if (!shared) {
> +			ntb->epf->bar[barno].barno = barno;
> +			ntb->epf->bar[barno].size = size;
> +			ntb->epf->bar[barno].addr = NULL;
> +			ntb->epf->bar[barno].phys_addr = 0;
> +			ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
> +					PCI_BASE_ADDRESS_MEM_TYPE_64 :
> +					PCI_BASE_ADDRESS_MEM_TYPE_32;
> +
> +			ret = pci_epc_set_bar(ntb->epf->epc,
> +					      ntb->epf->func_no,
> +					      ntb->epf->vfunc_no,
> +					      &ntb->epf->bar[barno]);
> +			if (ret) {
> +				dev_err(dev, "MW set failed\n");
> +				goto err_alloc_mem;
> +			}
>  		}
>
>  		/* Allocate EPC outbound memory windows to vpci vntb device */
> @@ -1310,10 +1408,11 @@ static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
>  	return ret;
>
>  err_set_bar:
> -	pci_epc_clear_bar(ntb->epf->epc,
> -			  ntb->epf->func_no,
> -			  ntb->epf->vfunc_no,
> -			  &ntb->epf->bar[barno]);
> +	if (!shared)
> +		pci_epc_clear_bar(ntb->epf->epc,
> +				  ntb->epf->func_no,
> +				  ntb->epf->vfunc_no,
> +				  &ntb->epf->bar[barno]);
>  err_alloc_mem:
>  	epf_ntb_mw_bar_clear(ntb, i);
>  	return ret;
> @@ -1331,10 +1430,11 @@ static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)
>
>  	for (i = 0; i < num_mws; i++) {
>  		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
> -		pci_epc_clear_bar(ntb->epf->epc,
> -				  ntb->epf->func_no,
> -				  ntb->epf->vfunc_no,
> -				  &ntb->epf->bar[barno]);
> +		if (!epf_ntb_dma_shares_bar(ntb, barno))
> +			pci_epc_clear_bar(ntb->epf->epc,
> +					  ntb->epf->func_no,
> +					  ntb->epf->vfunc_no,
> +					  &ntb->epf->bar[barno]);
>
>  		pci_epc_mem_free_addr(ntb->epf->epc,
>  				      ntb->vpci_mw_phy[i],
> @@ -1936,6 +2036,28 @@ static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,
>  	dev = &ndev->dev;
>  	barno = ntb->epf_ntb_bar[BAR_MW1 + idx];
>  	epf_bar = &ntb->epf->bar[barno];
> +	if (epf_ntb_dma_shares_bar(ntb, barno)) {
> +		guard(mutex)(&ntb->dma->lock);
> +
> +		if (ntb->dma->mw_fallback) {
> +			if (size > epf_bar->size)
> +				return -EINVAL;
> +		} else if (size != ntb->mws_size[idx]) {
> +			/*
> +			 * The MW submap spans the full size configured
> +			 * through configfs. DMA submaps follow it at
> +			 * fixed BAR offsets. Supporting smaller MWs
> +			 * would need a scratch-backed submap to fill
> +			 * the gap. Let's add that complexity only when
> +			 * it is really needed.
> +			 */
> +			return -EINVAL;
> +		}
> +
> +		return epf_ntb_dma_set_bar_locked(ntb, !ntb->dma->mw_fallback,
> +						  &addr);
> +	}
> +
>  	epf_bar->phys_addr = addr;
>  	epf_bar->barno = barno;
>  	epf_bar->size = size;
> @@ -2134,15 +2256,23 @@ static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,
>  			resource_size_t *size_max)
>  {
>  	struct epf_ntb *ntb = ntb_ndev(ndev);
> +	enum pci_barno barno = ntb->epf_ntb_bar[BAR_MW1 + idx];
> +	resource_size_t aalign = SZ_4K;
> +	u64 size = ntb->mws_size[idx];
> +
> +	if (epf_ntb_dma_shares_bar(ntb, barno) && ntb->dma->mw_fallback) {
> +		size = ntb->epf->bar[barno].size;
> +		aalign = size;
> +	}
>
>  	if (addr_align)
> -		*addr_align = SZ_4K;
> +		*addr_align = max_t(resource_size_t, SZ_4K, aalign);
>
>  	if (size_align)
>  		*size_align = 1;
>
>  	if (size_max)
> -		*size_max = ntb->mws_size[idx];
> +		*size_max = size;
>
>  	return 0;
>  }
> diff --git a/include/linux/pci-epf-ntb.h b/include/linux/pci-epf-ntb.h
> index 5cc0f0badae0..8131ed4c6ac9 100644
> --- a/include/linux/pci-epf-ntb.h
> +++ b/include/linux/pci-epf-ntb.h
> @@ -24,6 +24,9 @@
>  #define EPF_NTB_DB_COUNT_MASK			GENMASK(15, 0)
>  #define EPF_NTB_MSIX_ENABLE			BIT(16)
>
> +/* LINK_UP argument: the host honors the MW/DMA BAR boundary. */
> +#define EPF_NTB_LINK_UP_DMA_MW_SUPPORTED		BIT(0)
> +
>  #define EPF_NTB_COMMAND_STATUS_OK		1
>  #define EPF_NTB_COMMAND_STATUS_ERROR		2
>
> --
> 2.54.0
>