Re: [RFC PATCH v3 02/35] NTB: epf: Add mwN_offset support and config region versioning

[Frank Li <Frank.li@nxp.com>]

On Thu, Dec 18, 2025 at 12:15:36AM +0900, Koichiro Den wrote:
> Introduce new mwN_offset configfs attributes to specify memory window
> offsets. This enables mapping multiple windows into a single BAR at
> arbitrary offsets, improving layout flexibility.
>
> Extend the control register region and add a 32-bit config version
> field. Reuse NTB_EPF_TOPOLOGY (0x0C), which is currently unused, as the
> version register. The endpoint function driver writes 1
> (NTB_EPF_CTRL_VERSION_V1), and ntb_hw_epf reads it at probe time and
> refuses to bind to unknown versions.
>
> Endpoint running with an older kernel that do not program

Is it zero if EP have not program it?

> NTB_EPF_CTRL_VERSION will be rejected early by host with newer kernel,
> instead of misbehaving at runtime.

If old one is 0, try best to compatible with old version.

Frank
>
> Signed-off-by: Koichiro Den <den@valinux.co.jp>
> ---
>  drivers/ntb/hw/epf/ntb_hw_epf.c               |  44 +++++-
>  drivers/pci/endpoint/functions/pci-epf-vntb.c | 136 ++++++++++++++++--
>  2 files changed, 160 insertions(+), 20 deletions(-)
>
> diff --git a/drivers/ntb/hw/epf/ntb_hw_epf.c b/drivers/ntb/hw/epf/ntb_hw_epf.c
> index d3ecf25a5162..126ba38e32ea 100644
> --- a/drivers/ntb/hw/epf/ntb_hw_epf.c
> +++ b/drivers/ntb/hw/epf/ntb_hw_epf.c
> @@ -30,18 +30,22 @@
>  #define NTB_EPF_LINK_STATUS	0x0A
>  #define LINK_STATUS_UP		BIT(0)
>
> -#define NTB_EPF_TOPOLOGY	0x0C
> +/* 0x24 (32bit) is unused */
> +#define NTB_EPF_CTRL_VERSION	0x0C
>  #define NTB_EPF_LOWER_ADDR	0x10
>  #define NTB_EPF_UPPER_ADDR	0x14
>  #define NTB_EPF_LOWER_SIZE	0x18
>  #define NTB_EPF_UPPER_SIZE	0x1C
>  #define NTB_EPF_MW_COUNT	0x20
> -#define NTB_EPF_MW1_OFFSET	0x24
>  #define NTB_EPF_SPAD_OFFSET	0x28
>  #define NTB_EPF_SPAD_COUNT	0x2C
>  #define NTB_EPF_DB_ENTRY_SIZE	0x30
>  #define NTB_EPF_DB_DATA(n)	(0x34 + (n) * 4)
>  #define NTB_EPF_DB_OFFSET(n)	(0xB4 + (n) * 4)
> +#define NTB_EPF_MW_OFFSET(n)	(0x134 + (n) * 4)
> +#define NTB_EPF_MW_SIZE(n)	(0x144 + (n) * 4)
> +
> +#define NTB_EPF_CTRL_VERSION_V1	1
>
>  #define NTB_EPF_MIN_DB_COUNT	3
>  #define NTB_EPF_MAX_DB_COUNT	31
> @@ -451,11 +455,12 @@ static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
>  				    phys_addr_t *base, resource_size_t *size)
>  {
>  	struct ntb_epf_dev *ndev = ntb_ndev(ntb);
> -	u32 offset = 0;
> +	resource_size_t bar_sz;
> +	u32 offset, sz;
>  	int bar;
>
> -	if (idx == 0)
> -		offset = readl(ndev->ctrl_reg + NTB_EPF_MW1_OFFSET);
> +	offset = readl(ndev->ctrl_reg + NTB_EPF_MW_OFFSET(idx));
> +	sz = readl(ndev->ctrl_reg + NTB_EPF_MW_SIZE(idx));
>
>  	bar = ntb_epf_mw_to_bar(ndev, idx);
>  	if (bar < 0)
> @@ -464,8 +469,11 @@ static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
>  	if (base)
>  		*base = pci_resource_start(ndev->ntb.pdev, bar) + offset;
>
> -	if (size)
> -		*size = pci_resource_len(ndev->ntb.pdev, bar) - offset;
> +	if (size) {
> +		bar_sz = pci_resource_len(ndev->ntb.pdev, bar);
> +		*size = sz ? min_t(resource_size_t, sz, bar_sz - offset)
> +			   : (bar_sz > offset ? bar_sz - offset : 0);
> +	}
>
>  	return 0;
>  }
> @@ -547,6 +555,24 @@ static inline void ntb_epf_init_struct(struct ntb_epf_dev *ndev,
>  	ndev->ntb.ops = &ntb_epf_ops;
>  }
>
> +static int ntb_epf_check_version(struct ntb_epf_dev *ndev)
> +{
> +	struct device *dev = ndev->dev;
> +	u32 ver;
> +
> +	ver = readl(ndev->ctrl_reg + NTB_EPF_CTRL_VERSION);
> +
> +	switch (ver) {
> +	case NTB_EPF_CTRL_VERSION_V1:
> +		break;
> +	default:
> +		dev_err(dev, "Unsupported NTB EPF version %u\n", ver);
> +		return -EINVAL;
> +	}
> +
> +	return 0;
> +}
> +
>  static int ntb_epf_init_dev(struct ntb_epf_dev *ndev)
>  {
>  	struct device *dev = ndev->dev;
> @@ -695,6 +721,10 @@ static int ntb_epf_pci_probe(struct pci_dev *pdev,
>  		return ret;
>  	}
>
> +	ret = ntb_epf_check_version(ndev);
> +	if (ret)
> +		return ret;
> +
>  	ret = ntb_epf_init_dev(ndev);
>  	if (ret) {
>  		dev_err(dev, "Failed to init device\n");
> diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
> index 56aab5d354d6..4dfb3e40dffa 100644
> --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
> +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
> @@ -39,6 +39,7 @@
>  #include <linux/atomic.h>
>  #include <linux/delay.h>
>  #include <linux/io.h>
> +#include <linux/log2.h>
>  #include <linux/module.h>
>  #include <linux/slab.h>
>
> @@ -61,6 +62,7 @@ static struct workqueue_struct *kpcintb_workqueue;
>
>  #define LINK_STATUS_UP			BIT(0)
>
> +#define CTRL_VERSION			1
>  #define SPAD_COUNT			64
>  #define DB_COUNT			4
>  #define NTB_MW_OFFSET			2
> @@ -107,7 +109,7 @@ struct epf_ntb_ctrl {
>  	u32 argument;
>  	u16 command_status;
>  	u16 link_status;
> -	u32 topology;
> +	u32 version;
>  	u64 addr;
>  	u64 size;
>  	u32 num_mws;
> @@ -117,6 +119,8 @@ struct epf_ntb_ctrl {
>  	u32 db_entry_size;
>  	u32 db_data[MAX_DB_COUNT];
>  	u32 db_offset[MAX_DB_COUNT];
> +	u32 mw_offset[MAX_MW];
> +	u32 mw_size[MAX_MW];
>  } __packed;
>
>  struct epf_ntb {
> @@ -128,6 +132,7 @@ struct epf_ntb {
>  	u32 db_count;
>  	u32 spad_count;
>  	u64 mws_size[MAX_MW];
> +	u64 mws_offset[MAX_MW];
>  	atomic64_t db;
>  	u32 vbus_number;
>  	u16 vntb_pid;
> @@ -454,10 +459,13 @@ static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
>  	ntb->reg = base;
>
>  	ctrl = ntb->reg;
> +	ctrl->version = CTRL_VERSION;
>  	ctrl->spad_offset = ctrl_size;
>
>  	ctrl->spad_count = spad_count;
>  	ctrl->num_mws = ntb->num_mws;
> +	memset(ctrl->mw_offset, 0, sizeof(ctrl->mw_offset));
> +	memset(ctrl->mw_size, 0, sizeof(ctrl->mw_size));
>  	ntb->spad_size = spad_size;
>
>  	ctrl->db_entry_size = sizeof(u32);
> @@ -689,15 +697,31 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
>   */
>  static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
>  {
> +	struct device *dev = &ntb->epf->dev;
> +	u64 bar_ends[BAR_5 + 1] = { 0 };
> +	unsigned long bars_used = 0;
> +	enum pci_barno barno;
> +	u64 off, size, end;
>  	int ret = 0;
>  	int i;
> -	u64 size;
> -	enum pci_barno barno;
> -	struct device *dev = &ntb->epf->dev;
>
>  	for (i = 0; i < ntb->num_mws; i++) {
> -		size = ntb->mws_size[i];
>  		barno = ntb->epf_ntb_bar[BAR_MW1 + i];
> +		off = ntb->mws_offset[i];
> +		size = ntb->mws_size[i];
> +		end = off + size;
> +		if (end > bar_ends[barno])
> +			bar_ends[barno] = end;
> +		bars_used |= BIT(barno);
> +	}
> +
> +	for (barno = BAR_0; barno <= BAR_5; barno++) {
> +		if (!(bars_used & BIT(barno)))
> +			continue;
> +		if (bar_ends[barno] < SZ_4K)
> +			size = SZ_4K;
> +		else
> +			size = roundup_pow_of_two(bar_ends[barno]);
>
>  		ntb->epf->bar[barno].barno = barno;
>  		ntb->epf->bar[barno].size = size;
> @@ -713,8 +737,12 @@ static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
>  				      &ntb->epf->bar[barno]);
>  		if (ret) {
>  			dev_err(dev, "MW set failed\n");
> -			goto err_alloc_mem;
> +			goto err_set_bar;
>  		}
> +	}
> +
> +	for (i = 0; i < ntb->num_mws; i++) {
> +		size = ntb->mws_size[i];
>
>  		/* Allocate EPC outbound memory windows to vpci vntb device */
>  		ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc,
> @@ -723,19 +751,31 @@ static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
>  		if (!ntb->vpci_mw_addr[i]) {
>  			ret = -ENOMEM;
>  			dev_err(dev, "Failed to allocate source address\n");
> -			goto err_set_bar;
> +			goto err_alloc_mem;
>  		}
>  	}
>
> +	for (i = 0; i < ntb->num_mws; i++) {
> +		ntb->reg->mw_offset[i] = (u32)ntb->mws_offset[i];
> +		ntb->reg->mw_size[i] = (u32)ntb->mws_size[i];
> +	}
> +
>  	return ret;
>
> -err_set_bar:
> -	pci_epc_clear_bar(ntb->epf->epc,
> -			  ntb->epf->func_no,
> -			  ntb->epf->vfunc_no,
> -			  &ntb->epf->bar[barno]);
>  err_alloc_mem:
> -	epf_ntb_mw_bar_clear(ntb, i);
> +	while (--i >= 0)
> +		pci_epc_mem_free_addr(ntb->epf->epc,
> +				      ntb->vpci_mw_phy[i],
> +				      ntb->vpci_mw_addr[i],
> +				      ntb->mws_size[i]);
> +err_set_bar:
> +	while (--barno >= BAR_0)
> +		if (bars_used & BIT(barno))
> +			pci_epc_clear_bar(ntb->epf->epc,
> +					  ntb->epf->func_no,
> +					  ntb->epf->vfunc_no,
> +					  &ntb->epf->bar[barno]);
> +
>  	return ret;
>  }
>
> @@ -1040,6 +1080,60 @@ static ssize_t epf_ntb_##_name##_store(struct config_item *item,	\
>  	return len;							\
>  }
>
> +#define EPF_NTB_MW_OFF_R(_name)						\
> +static ssize_t epf_ntb_##_name##_show(struct config_item *item,		\
> +				      char *page)			\
> +{									\
> +	struct config_group *group = to_config_group(item);		\
> +	struct epf_ntb *ntb = to_epf_ntb(group);			\
> +	struct device *dev = &ntb->epf->dev;				\
> +	int win_no, idx;						\
> +									\
> +	if (sscanf(#_name, "mw%d_offset", &win_no) != 1)		\
> +		return -EINVAL;						\
> +									\
> +	idx = win_no - 1;						\
> +	if (idx < 0 || idx >= ntb->num_mws) {				\
> +		dev_err(dev, "MW%d out of range (num_mws=%d)\n",	\
> +			win_no, ntb->num_mws);				\
> +		return -EINVAL;						\
> +	}								\
> +									\
> +	idx = array_index_nospec(idx, ntb->num_mws);			\
> +	return sprintf(page, "%llu\n", ntb->mws_offset[idx]);		\
> +}
> +
> +#define EPF_NTB_MW_OFF_W(_name)						\
> +static ssize_t epf_ntb_##_name##_store(struct config_item *item,	\
> +				       const char *page, size_t len)	\
> +{									\
> +	struct config_group *group = to_config_group(item);		\
> +	struct epf_ntb *ntb = to_epf_ntb(group);			\
> +	struct device *dev = &ntb->epf->dev;				\
> +	int win_no, idx;						\
> +	u64 val;							\
> +	int ret;							\
> +									\
> +	ret = kstrtou64(page, 0, &val);					\
> +	if (ret)							\
> +		return ret;						\
> +									\
> +	if (sscanf(#_name, "mw%d_offset", &win_no) != 1)		\
> +		return -EINVAL;						\
> +									\
> +	idx = win_no - 1;						\
> +	if (idx < 0 || idx >= ntb->num_mws) {				\
> +		dev_err(dev, "MW%d out of range (num_mws=%d)\n",	\
> +			win_no, ntb->num_mws);				\
> +		return -EINVAL;						\
> +	}								\
> +									\
> +	idx = array_index_nospec(idx, ntb->num_mws);			\
> +	ntb->mws_offset[idx] = val;					\
> +									\
> +	return len;							\
> +}
> +
>  #define EPF_NTB_BAR_R(_name, _id)					\
>  	static ssize_t epf_ntb_##_name##_show(struct config_item *item,	\
>  					      char *page)		\
> @@ -1110,6 +1204,14 @@ EPF_NTB_MW_R(mw3)
>  EPF_NTB_MW_W(mw3)
>  EPF_NTB_MW_R(mw4)
>  EPF_NTB_MW_W(mw4)
> +EPF_NTB_MW_OFF_R(mw1_offset)
> +EPF_NTB_MW_OFF_W(mw1_offset)
> +EPF_NTB_MW_OFF_R(mw2_offset)
> +EPF_NTB_MW_OFF_W(mw2_offset)
> +EPF_NTB_MW_OFF_R(mw3_offset)
> +EPF_NTB_MW_OFF_W(mw3_offset)
> +EPF_NTB_MW_OFF_R(mw4_offset)
> +EPF_NTB_MW_OFF_W(mw4_offset)
>  EPF_NTB_BAR_R(ctrl_bar, BAR_CONFIG)
>  EPF_NTB_BAR_W(ctrl_bar, BAR_CONFIG)
>  EPF_NTB_BAR_R(db_bar, BAR_DB)
> @@ -1130,6 +1232,10 @@ CONFIGFS_ATTR(epf_ntb_, mw1);
>  CONFIGFS_ATTR(epf_ntb_, mw2);
>  CONFIGFS_ATTR(epf_ntb_, mw3);
>  CONFIGFS_ATTR(epf_ntb_, mw4);
> +CONFIGFS_ATTR(epf_ntb_, mw1_offset);
> +CONFIGFS_ATTR(epf_ntb_, mw2_offset);
> +CONFIGFS_ATTR(epf_ntb_, mw3_offset);
> +CONFIGFS_ATTR(epf_ntb_, mw4_offset);
>  CONFIGFS_ATTR(epf_ntb_, vbus_number);
>  CONFIGFS_ATTR(epf_ntb_, vntb_pid);
>  CONFIGFS_ATTR(epf_ntb_, vntb_vid);
> @@ -1148,6 +1254,10 @@ static struct configfs_attribute *epf_ntb_attrs[] = {
>  	&epf_ntb_attr_mw2,
>  	&epf_ntb_attr_mw3,
>  	&epf_ntb_attr_mw4,
> +	&epf_ntb_attr_mw1_offset,
> +	&epf_ntb_attr_mw2_offset,
> +	&epf_ntb_attr_mw3_offset,
> +	&epf_ntb_attr_mw4_offset,
>  	&epf_ntb_attr_vbus_number,
>  	&epf_ntb_attr_vntb_pid,
>  	&epf_ntb_attr_vntb_vid,
> --
> 2.51.0
>