Re: [PATCH v3] ocxl: control via sysfs whether the FPGA is reloaded on a link reset

[Andrew Donnellan <ajd@linux.ibm.com>]

On 18/3/20 9:02 pm, Frederic Barrat wrote:
> From: Philippe Bergheaud <felix@linux.ibm.com>
> 
> Some opencapi FPGA images allow to control if the FPGA should be reloaded
> on the next adapter reset. If it is supported, the image specifies it
> through a Vendor Specific DVSEC in the config space of function 0.
> 
> Signed-off-by: Philippe Bergheaud <felix@linux.ibm.com>

As I've raised privately - I think we need an additional identifier 
within the Vendor-Specific DVSEC to distinguish between the IBM 
reference implementation of the CFG subsystem and alternative 
implementations that may use the Vendor-Specific DVSEC for other purposes.

Further comments below.

> diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c
> index c8e19bfb5ef9..b364b6ceb996 100644
> --- a/drivers/misc/ocxl/config.c
> +++ b/drivers/misc/ocxl/config.c
> @@ -71,6 +71,20 @@ static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx)
>   	return 0;
>   }
>   
> +/**
> + * get_function_0() - Find a related PCI device (function 0)
> + * @device: PCI device to match
> + *
> + * Returns a pointer to the related device, or null if not found
> + */
> +static struct pci_dev *get_function_0(struct pci_dev *dev)
> +{
> +	unsigned int devfn = PCI_DEVFN(PCI_SLOT(dev->devfn), 0);
> +
> +	return pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
> +					   dev->bus->number, devfn);
> +}
> +
>   static void read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn)
>   {
>   	u16 val;
> @@ -159,7 +173,7 @@ static int read_dvsec_afu_info(struct pci_dev *dev, struct ocxl_fn_config *fn)
>   static int read_dvsec_vendor(struct pci_dev *dev)
>   {
>   	int pos;
> -	u32 cfg, tlx, dlx;
> +	u32 cfg, tlx, dlx, reset_reload;
>   
>   	/*
>   	 * vendor specific DVSEC is optional
> @@ -183,6 +197,58 @@ static int read_dvsec_vendor(struct pci_dev *dev)
>   	dev_dbg(&dev->dev, "  CFG version = 0x%x\n", cfg);
>   	dev_dbg(&dev->dev, "  TLX version = 0x%x\n", tlx);
>   	dev_dbg(&dev->dev, "  DLX version = 0x%x\n", dlx);
> +
> +	if (ocxl_config_get_reset_reload(dev, &reset_reload) != 0)
> +		dev_dbg(&dev->dev, "  ResetReload is not available\n");
> +	else
> +		dev_dbg(&dev->dev, "  ResetReload = 0x%x\n", reset_reload);
> +	return 0;
> +}
> +
> +int ocxl_config_get_reset_reload(struct pci_dev *dev, int *val)
> +{
> +	int reset_reload = -1;

Would prefer if this was a u32, to match the type in the 
pci_read_config_dword() signature

> +	int pos = 0;
> +	struct pci_dev *dev0 = dev;
> +
> +	if (PCI_FUNC(dev->devfn) != 0)
> +		dev0 = get_function_0(dev);
> +
> +	if (dev0)
> +		pos = find_dvsec(dev0, OCXL_DVSEC_VENDOR_ID);
> +
> +	if (pos)
> +		pci_read_config_dword(dev0,
> +				      pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
> +				      &reset_reload);
> +	if (reset_reload == -1)
> +		return reset_reload;

Can we safely assume that 0xFFFFFFFF is never going to be a valid value 
for this dword? The document I'm looking at only states that bits 31:1 
are reserved, not that any of them are guaranteed to be a 0 in a future 
revision.

> +
> +	*val = reset_reload & BIT(0);
> +	return 0;
> +}
> +
> +int ocxl_config_set_reset_reload(struct pci_dev *dev, int val)
> +{
> +	int reset_reload = -1;
> +	int pos = 0;
> +	struct pci_dev *dev0 = dev;
> +
> +	if (PCI_FUNC(dev->devfn) != 0)
> +		dev0 = get_function_0(dev);
> +
> +	if (dev0)
> +		pos = find_dvsec(dev0, OCXL_DVSEC_VENDOR_ID);
> +
> +	if (pos)
> +		pci_read_config_dword(dev0,
> +				      pos + OCXL_DVSEC_VENDOR_RESET_RELOAD,
> +				      &reset_reload);
> +	if (reset_reload == -1)
> +		return reset_reload;
> +
> +	val &= BIT(0);

I think we want to keep the existing value of reserved bits.

Something like

	val = (val & BIT(0)) | (reset_reload & ~BIT(0));

?

> +	pci_write_config_dword(dev0, pos + OCXL_DVSEC_VENDOR_RESET_RELOAD, val);
>   	return 0;
>   }
>   
> diff --git a/drivers/misc/ocxl/ocxl_internal.h b/drivers/misc/ocxl/ocxl_internal.h
> index 345bf843a38e..af9a84aeee6f 100644
> --- a/drivers/misc/ocxl/ocxl_internal.h
> +++ b/drivers/misc/ocxl/ocxl_internal.h
> @@ -112,6 +112,12 @@ void ocxl_actag_afu_free(struct ocxl_fn *fn, u32 start, u32 size);
>    */
>   int ocxl_config_get_pasid_info(struct pci_dev *dev, int *count);
>   
> +/*
> + * Control whether the FPGA is reloaded on a link reset
> + */
> +int ocxl_config_get_reset_reload(struct pci_dev *dev, int *val);
> +int ocxl_config_set_reset_reload(struct pci_dev *dev, int val);
> +
>   /*
>    * Check if an AFU index is valid for the given function.
>    *
> diff --git a/drivers/misc/ocxl/sysfs.c b/drivers/misc/ocxl/sysfs.c
> index 58f1ba264206..8f69f7311343 100644
> --- a/drivers/misc/ocxl/sysfs.c
> +++ b/drivers/misc/ocxl/sysfs.c
> @@ -51,11 +51,46 @@ static ssize_t contexts_show(struct device *device,
>   			afu->pasid_count, afu->pasid_max);
>   }
>   
> +static ssize_t reload_on_reset_show(struct device *device,
> +		struct device_attribute *attr,
> +		char *buf)
> +{
> +	struct ocxl_afu *afu = to_afu(device);
> +	struct ocxl_fn *fn = afu->fn;
> +	struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
> +	int val;
> +
> +	if (ocxl_config_get_reset_reload(pci_dev, &val))
> +		return scnprintf(buf, PAGE_SIZE, "unavailable\n");
> +
> +	return scnprintf(buf, PAGE_SIZE, "%d\n", val);
> +}
> +
> +static ssize_t reload_on_reset_store(struct device *device,
> +		struct device_attribute *attr,
> +		const char *buf, size_t count)
> +{
> +	struct ocxl_afu *afu = to_afu(device);
> +	struct ocxl_fn *fn = afu->fn;
> +	struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
> +	int rc, val;
> +
> +	rc = sscanf(buf, "%i", &val);

Checkpatch suggests turning this into kstrtoint()

https://openpower.xyz/job/snowpatch/job/snowpatch-linux-checkpatch/12312//artifact/linux/checkpatch.log

> +	if ((rc != 1) || !(val == 1 || val == 0))
> +		return -EINVAL;
> +
> +	if (ocxl_config_set_reset_reload(pci_dev, val))
> +		return -ENODEV;
> +
> +	return count;
> +}
> +
>   static struct device_attribute afu_attrs[] = {
>   	__ATTR_RO(global_mmio_size),
>   	__ATTR_RO(pp_mmio_size),
>   	__ATTR_RO(afu_version),
>   	__ATTR_RO(contexts),
> +	__ATTR_RW(reload_on_reset),
>   };
>   
>   static ssize_t global_mmio_read(struct file *filp, struct kobject *kobj,
> diff --git a/include/misc/ocxl-config.h b/include/misc/ocxl-config.h
> index 3526fa996a22..ccfd3b463517 100644
> --- a/include/misc/ocxl-config.h
> +++ b/include/misc/ocxl-config.h
> @@ -41,5 +41,6 @@
>   #define   OCXL_DVSEC_VENDOR_CFG_VERS            0x0C
>   #define   OCXL_DVSEC_VENDOR_TLX_VERS            0x10
>   #define   OCXL_DVSEC_VENDOR_DLX_VERS            0x20
> +#define   OCXL_DVSEC_VENDOR_RESET_RELOAD        0x38
>   
>   #endif /* _OCXL_CONFIG_H_ */
> 

-- 
Andrew Donnellan              OzLabs, ADL Canberra
ajd@linux.ibm.com             IBM Australia Limited