Re: [RFC PATCH 1/2] powerpc/powernv: Add support for CXL mode switch that need PHB reset

[christophe lombard <clombard@linux.vnet.ibm.com>]

On 25/01/2019 06:11, Vaibhav Jain wrote:
> Recent updates to OPAL [1] have provided support for new CXL modes on
> PHB that need to force a cold reset on the bridge (CRESET). However
> PHB CRESET is a multi step process and cannot be completed
> synchronously as expected by current kernel implementation that issues
> opal call opal_pci_set_phb_cxl_mode().
> 
> Hence this patch updates pnv_phb_to_cxl_mode() to implement a polling
> loop that handles specific error codes (OPAL_BUSY) returned from
> opal_pci_set_phb_cxl_mode() and drive the OPAL pci-state machine, if the
> requested CXL mode needs a CRESET.
> 
> The patch also updates pnv_phb_to_cxl_mode() to convert and return
> OPAL error codes into kernel error codes. This removes a previous
> issue where callers to this function would have to include
> 'opal-api.h' to check for specific OPAL error codes.
> 
> References:
> [1]: https://lists.ozlabs.org/pipermail/skiboot/2019-January/013063.html
> 
> Signed-off-by: Vaibhav Jain <vaibhav@linux.ibm.com>
> ---
>   arch/powerpc/platforms/powernv/pci-cxl.c | 71 +++++++++++++++++++++---
>   1 file changed, 63 insertions(+), 8 deletions(-)
> 
> diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c
> index 1b18111453d7..d33d662c6212 100644
> --- a/arch/powerpc/platforms/powernv/pci-cxl.c
> +++ b/arch/powerpc/platforms/powernv/pci-cxl.c
> @@ -10,6 +10,7 @@
>   #include <linux/module.h>
>   #include <asm/pnv-pci.h>
>   #include <asm/opal.h>
> +#include <linux/delay.h>
> 
>   #include "pci.h"
> 
> @@ -18,21 +19,75 @@ int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode)
>   	struct pci_controller *hose = pci_bus_to_host(dev->bus);
>   	struct pnv_phb *phb = hose->private_data;
>   	struct pnv_ioda_pe *pe;
> +	unsigned long starttime, endtime;
>   	int rc;
> 
>   	pe = pnv_ioda_get_pe(dev);
>   	if (!pe)
> -		return -ENODEV;
> +		return -ENOENT;

The return code of pnv_phb_to_cxl_mode() is also returned by an api in 
the cxllib librarie. So, hoping that nobody test the value !!

> +
> +	pe_info(pe, "Switching PHB to CXL mode=%d\n", mode);
> +
> +	/*
> +	 * Use a 15 second timeout for mode switch. Value arrived after
> +	 * limited testing and may need more tweaking.
> +	 */
> +	starttime = jiffies;
> +	endtime = starttime + HZ * 15;
> +
> +	do {
> +		rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode,
> +					       pe->pe_number);
> +
> +		/* Wait until mode transistion done */
> +		if (rc != OPAL_BUSY && rc != OPAL_BUSY_EVENT)
> +			break;
> +
> +		/* Check if we timedout */
> +		if (time_after(jiffies, endtime)) {
> +			rc = OPAL_TIMEOUT;
> +			break;
> +		}
> 
> -	pe_info(pe, "Switching PHB to CXL\n");
> +		/* Opal Busy with mode switch. Run pci state-machine */
> +		rc = opal_pci_poll(phb->opal_id);
> +		if (rc >= 0) {
> +			/* wait for some time */
> +			if (rc > 0)
> +				msleep(rc);
> +			opal_poll_events(NULL);
> +			rc = OPAL_BUSY;
> +			/* Continue with the mode switch */
> +		}
> +	} while (rc == OPAL_BUSY || rc == OPAL_BUSY_EVENT);
> +
> +	pe_level_printk(pe, KERN_DEBUG, "CXL mode switch finished in %u-msecs.",
> +			jiffies_to_msecs(jiffies - starttime));
> 
> -	rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number);
> -	if (rc == OPAL_UNSUPPORTED)
> -		dev_err(&dev->dev, "Required cxl mode not supported by firmware - update skiboot\n");
> -	else if (rc)
> -		dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc);
> +	/* Check OPAL errors and convert them to kernel error codes */
> +	switch (rc) {
> +	case OPAL_SUCCESS:
> +		return 0;
> 
> -	return rc;
> +	case OPAL_PARAMETER:
> +		dev_err(&dev->dev, "CXL not supported on this PHB\n");
> +		return -ENOENT;
> +
> +	case OPAL_UNSUPPORTED:
> +		dev_err(&dev->dev,
> +			"Required cxl mode not supported by firmware"
> +			" - update skiboot\n");
> +		return -ENODEV;
> +
> +	case OPAL_TIMEOUT:
> +		dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode Timedout\n");
> +		return -ETIME;
> +
> +	default:
> +		dev_err(&dev->dev,
> +			"opal_pci_set_phb_cxl_mode failed: %i\n", rc);
> +		return -EIO;
> +	};
>   }
>   EXPORT_SYMBOL(pnv_phb_to_cxl_mode);
>