* Re: [patch] powerpc/spufs: remove duplicate SPUFS_CNTL_MAP_SIZE define
From: Jeremy Kerr @ 2014-06-09 23:09 UTC (permalink / raw)
To: Dan Carpenter
Cc: cbe-oss-dev, Arnd Bergmann, kernel-janitors, Paul Mackerras,
linuxppc-dev
In-Reply-To: <20140609151917.GM9600@mwanda>
Hi Dan,
> The SPUFS_CNTL_MAP_SIZE define is cut and pasted twice so we can delete
> the second instance.
Looks good to me.
Acked-by: Jeremy Kerr <jk@ozlabs.org>
Cheers,
Jeremy
^ permalink raw reply
* Re: [PATCH v9 3/3] drivers/vfio: EEH support for VFIO PCI device
From: Gavin Shan @ 2014-06-09 23:49 UTC (permalink / raw)
To: Alex Williamson; +Cc: aik, Gavin Shan, kvm-ppc, agraf, qiudayu, linuxppc-dev
In-Reply-To: <1402078804.14174.24.camel@ul30vt.home>
On Fri, Jun 06, 2014 at 12:20:04PM -0600, Alex Williamson wrote:
>On Fri, 2014-06-06 at 15:00 +1000, Gavin Shan wrote:
>> The patch adds new IOCTL commands for sPAPR VFIO container device
>> to support EEH functionality for PCI devices, which have been passed
>> through from host to somebody else via VFIO.
>>
>> Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
>> Acked-by: Alexander Graf <agraf@suse.de>
>> ---
>> Documentation/vfio.txt | 87 +++++++++++++++++++++++++++++++++++--
>> drivers/vfio/Makefile | 1 +
>> drivers/vfio/pci/vfio_pci.c | 18 ++++++--
>> drivers/vfio/vfio_iommu_spapr_tce.c | 17 +++++++-
>> drivers/vfio/vfio_spapr_eeh.c | 87 +++++++++++++++++++++++++++++++++++++
>> include/linux/vfio.h | 23 ++++++++++
>> include/uapi/linux/vfio.h | 34 +++++++++++++++
>> 7 files changed, 259 insertions(+), 8 deletions(-)
>> create mode 100644 drivers/vfio/vfio_spapr_eeh.c
>>
>> diff --git a/Documentation/vfio.txt b/Documentation/vfio.txt
>> index b9ca023..3fa4538 100644
>> --- a/Documentation/vfio.txt
>> +++ b/Documentation/vfio.txt
>> @@ -305,7 +305,15 @@ faster, the map/unmap handling has been implemented in real mode which provides
>> an excellent performance which has limitations such as inability to do
>> locked pages accounting in real time.
>>
>> -So 3 additional ioctls have been added:
>> +4) According to sPAPR specification, A Partitionable Endpoint (PE) is an I/O
>> +subtree that can be treated as a unit for the purposes of partitioning and
>> +error recovery. A PE may be a single or multi-function IOA (IO Adapter), a
>> +function of a multi-function IOA, or multiple IOAs (possibly including switch
>> +and bridge structures above the multiple IOAs). PPC64 guests detect PCI errors
>> +and recover from them via EEH RTAS services, which works on the basis of
>> +additional ioctl commands.
>> +
>> +So 4 additional ioctls have been added:
>>
>> VFIO_IOMMU_SPAPR_TCE_GET_INFO - returns the size and the start
>> of the DMA window on the PCI bus.
>> @@ -316,9 +324,12 @@ So 3 additional ioctls have been added:
>>
>> VFIO_IOMMU_DISABLE - disables the container.
>>
>> + VFIO_EEH_PE_OP - provides an API for EEH setup, error detection and recovery.
>>
>> The code flow from the example above should be slightly changed:
>>
>> + struct vfio_eeh_pe_op pe_op = { .argsz = sizeof(pe_op) };
>> +
>> .....
>> /* Add the group to the container */
>> ioctl(group, VFIO_GROUP_SET_CONTAINER, &container);
>> @@ -342,9 +353,79 @@ The code flow from the example above should be slightly changed:
>> dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
>>
>> /* Check here is .iova/.size are within DMA window from spapr_iommu_info */
>> -
>> ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map);
>> - .....
>> +
>> + /* Get a file descriptor for the device */
>> + device = ioctl(group, VFIO_GROUP_GET_DEVICE_FD, "0000:06:0d.0");
>> +
>> + ....
>> +
>> + /* Gratuitous device reset and go... */
>> + ioctl(device, VFIO_DEVICE_RESET);
>> +
>> + /* Make sure EEH is supported */
>> + ioctl(container, VFIO_CHECK_EXTENSION, VFIO_EEH);
>> +
>> + /* Enable the EEH functionality on the device */
>> + pe_op.op = VFIO_EEH_PE_ENABLE;
>> + ioctl(container, VFIO_EEH_PE_OP, &pe_op);
>> +
>> + /* You're suggested to create additional data struct to represent
>> + * PE, and put child devices belonging to same IOMMU group to the
>> + * PE instance for later reference.
>> + */
>> +
>> + /* Check the PE's state and make sure it's in functional state */
>> + pe_op.op = VFIO_EEH_PE_GET_STATE;
>> + ioctl(container, VFIO_EEH_PE_OP, &pe_op);
>> +
>> + /* Save device state using pci_save_state().
>> + * EEH should be enabled on the specified device.
>> + */
>> +
>> + ....
>> +
>> + /* When 0xFF's returned from reading PCI config space or IO BARs
>> + * of the PCI device. Check the PE's state to see if that has been
>> + * frozen.
>> + */
>> + ioctl(container, VFIO_EEH_PE_OP, &pe_op);
>> +
>> + /* Waiting for pending PCI transactions to be completed and don't
>> + * produce any more PCI traffic from/to the affected PE until
>> + * recovery is finished.
>> + */
>> +
>> + /* Enable IO for the affected PE and collect logs. Usually, the
>> + * standard part of PCI config space, AER registers are dumped
>> + * as logs for further analysis.
>> + */
>> + pe_op.op = VFIO_EEH_PE_UNFREEZE_IO;
>> + ioctl(container, VFIO_EEH_PE_OP, &pe_op);
>> +
>> + /*
>> + * Issue PE reset: hot or fundamental reset. Usually, hot reset
>> + * is enough. However, the firmware of some PCI adapters would
>> + * require fundamental reset.
>> + */
>> + pe_op.op = VFIO_EEH_PE_RESET_HOT;
>> + ioctl(container, VFIO_EEH_PE_OP, &pe_op);
>> + pe_op.op = VFIO_EEH_PE_RESET_DEACTIVATE;
>> + ioctl(container, VFIO_EEH_PE_OP, &pe_op);
>> +
>> + /* Configure the PCI bridges for the affected PE */
>> + pe_op.op = VFIO_EEH_PE_CONFIGURE;
>> + ioctl(container, VFIO_EEH_PE_OP, &pe_op);
>> +
>> + /* Restored state we saved at initialization time. pci_restore_state()
>> + * is good enough as an example.
>> + */
>> +
>> + /* Hopefully, error is recovered successfully. Now, you can resume to
>> + * start PCI traffic to/from the affected PE.
>> + */
>> +
>> + ....
>>
>> -------------------------------------------------------------------------------
>>
>> diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
>> index 72bfabc..50e30bc 100644
>> --- a/drivers/vfio/Makefile
>> +++ b/drivers/vfio/Makefile
>> @@ -1,4 +1,5 @@
>> obj-$(CONFIG_VFIO) += vfio.o
>> obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
>> obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
>> +obj-$(CONFIG_EEH) += vfio_spapr_eeh.o
>> obj-$(CONFIG_VFIO_PCI) += pci/
>> diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
>> index 7ba0424..0122665 100644
>> --- a/drivers/vfio/pci/vfio_pci.c
>> +++ b/drivers/vfio/pci/vfio_pci.c
>> @@ -156,8 +156,10 @@ static void vfio_pci_release(void *device_data)
>> {
>> struct vfio_pci_device *vdev = device_data;
>>
>> - if (atomic_dec_and_test(&vdev->refcnt))
>> + if (atomic_dec_and_test(&vdev->refcnt)) {
>> + vfio_spapr_pci_eeh_release(vdev->pdev);
>> vfio_pci_disable(vdev);
>> + }
>>
>> module_put(THIS_MODULE);
>> }
>> @@ -165,19 +167,27 @@ static void vfio_pci_release(void *device_data)
>> static int vfio_pci_open(void *device_data)
>> {
>> struct vfio_pci_device *vdev = device_data;
>> + int ret;
>>
>> if (!try_module_get(THIS_MODULE))
>> return -ENODEV;
>>
>> if (atomic_inc_return(&vdev->refcnt) == 1) {
>> - int ret = vfio_pci_enable(vdev);
>> + ret = vfio_pci_enable(vdev);
>> + if (ret)
>> + goto error;
>> +
>> + ret = vfio_spapr_pci_eeh_open(vdev->pdev);
>> if (ret) {
>> - module_put(THIS_MODULE);
>> - return ret;
>> + vfio_pci_disable(vdev);
>> + goto error;
>> }
>> }
>>
>> return 0;
>> +error:
>> + module_put(THIS_MODULE);
>> + return ret;
>> }
>>
>> static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
>> diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c
>> index a84788b..730b4ef 100644
>> --- a/drivers/vfio/vfio_iommu_spapr_tce.c
>> +++ b/drivers/vfio/vfio_iommu_spapr_tce.c
>> @@ -156,7 +156,16 @@ static long tce_iommu_ioctl(void *iommu_data,
>>
>> switch (cmd) {
>> case VFIO_CHECK_EXTENSION:
>> - return (arg == VFIO_SPAPR_TCE_IOMMU) ? 1 : 0;
>> + switch (arg) {
>> + case VFIO_SPAPR_TCE_IOMMU:
>> + ret = 1;
>> + break;
>> + default:
>> + ret = vfio_spapr_iommu_eeh_ioctl(NULL, cmd, arg);
>> + break;
>> + }
>> +
>> + return (ret < 0) ? 0 : ret;
>>
>> case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
>> struct vfio_iommu_spapr_tce_info info;
>> @@ -283,6 +292,12 @@ static long tce_iommu_ioctl(void *iommu_data,
>> tce_iommu_disable(container);
>> mutex_unlock(&container->lock);
>> return 0;
>> + case VFIO_EEH_PE_OP:
>> + if (!container->tbl || !container->tbl->it_group)
>> + return -ENODEV;
>> +
>> + return vfio_spapr_iommu_eeh_ioctl(container->tbl->it_group,
>> + cmd, arg);
>> }
>>
>> return -ENOTTY;
>> diff --git a/drivers/vfio/vfio_spapr_eeh.c b/drivers/vfio/vfio_spapr_eeh.c
>> new file mode 100644
>> index 0000000..d438394
>> --- /dev/null
>> +++ b/drivers/vfio/vfio_spapr_eeh.c
>> @@ -0,0 +1,87 @@
>> +/*
>> + * EEH functionality support for VFIO devices. The feature is only
>> + * available on sPAPR compatible platforms.
>> + *
>> + * Copyright Gavin Shan, IBM Corporation 2014.
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License version 2 as
>> + * published by the Free Software Foundation.
>> + */
>> +
>> +#include <linux/uaccess.h>
>> +#include <linux/vfio.h>
>> +#include <asm/eeh.h>
>> +
>> +/* We might build address mapping here for "fast" path later */
>> +int vfio_spapr_pci_eeh_open(struct pci_dev *pdev)
>> +{
>> + return eeh_dev_open(pdev);
>> +}
>> +
>> +void vfio_spapr_pci_eeh_release(struct pci_dev *pdev)
>> +{
>> + eeh_dev_release(pdev);
>> +}
>> +
>> +long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group,
>> + unsigned int cmd, unsigned long arg)
>> +{
>> + struct eeh_pe *pe;
>> + struct vfio_eeh_pe_op op;
>> + unsigned long minsz;
>> + long ret = -EINVAL;
>> +
>> + switch (cmd) {
>> + case VFIO_CHECK_EXTENSION:
>> + if (arg == VFIO_EEH)
>> + ret = eeh_enabled() ? 1 : 0;
>> + else
>> + ret = 0;
>> + break;
>> + case VFIO_EEH_PE_OP:
>> + pe = eeh_iommu_group_to_pe(group);
>> + if (!pe)
>> + return -ENODEV;
>> +
>> + minsz = offsetofend(struct vfio_eeh_pe_op, op);
>> + if (copy_from_user(&op, (void __user *)arg, minsz))
>> + return -EFAULT;
>> + if (op.argsz < minsz)
>> + return -EINVAL;
>
>I'm running low on comments, but I found one more. flags should be
>tested as zero here or else we'll run into problems with adding new
>functionality later. The caller could leave it uninitialized and pass
>junk which would then break if we start using those flags. New
>userspace on an old kernel could also intentionally pass flag bits that
>we ignore and let through here.
>
Thanks, Alex. I'll fix in next revision. The QEMU part needs corresponding
changes (set flags to zero), which will be updated and sent separately.
>
>How are you expecting this to go in, an ack from me then pushed through
>AlexG's tree? Thanks,
>
I don't think it can catch 3.16 merge window. Perhaps, Patch[1/3] and
Patch[2/3] goes to Ben's tree and Patch[3/3] could be picked by AlexG
or you, but I'm not sure.
Thanks,
Gavin
>> +
>> + switch (op.op) {
>> + case VFIO_EEH_PE_DISABLE:
>> + ret = eeh_pe_set_option(pe, EEH_OPT_DISABLE);
>> + break;
>> + case VFIO_EEH_PE_ENABLE:
>> + ret = eeh_pe_set_option(pe, EEH_OPT_ENABLE);
>> + break;
>> + case VFIO_EEH_PE_UNFREEZE_IO:
>> + ret = eeh_pe_set_option(pe, EEH_OPT_THAW_MMIO);
>> + break;
>> + case VFIO_EEH_PE_UNFREEZE_DMA:
>> + ret = eeh_pe_set_option(pe, EEH_OPT_THAW_DMA);
>> + break;
>> + case VFIO_EEH_PE_GET_STATE:
>> + ret = eeh_pe_get_state(pe);
>> + break;
>> + case VFIO_EEH_PE_RESET_DEACTIVATE:
>> + ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE);
>> + break;
>> + case VFIO_EEH_PE_RESET_HOT:
>> + ret = eeh_pe_reset(pe, EEH_RESET_HOT);
>> + break;
>> + case VFIO_EEH_PE_RESET_FUNDAMENTAL:
>> + ret = eeh_pe_reset(pe, EEH_RESET_FUNDAMENTAL);
>> + break;
>> + case VFIO_EEH_PE_CONFIGURE:
>> + ret = eeh_pe_configure(pe);
>> + break;
>> + default:
>> + ret = -EINVAL;
>> + }
>> + }
>> +
>> + return ret;
>> +}
>> diff --git a/include/linux/vfio.h b/include/linux/vfio.h
>> index 81022a52..0d3bb8f 100644
>> --- a/include/linux/vfio.h
>> +++ b/include/linux/vfio.h
>> @@ -99,4 +99,27 @@ extern int vfio_external_user_iommu_id(struct vfio_group *group);
>> extern long vfio_external_check_extension(struct vfio_group *group,
>> unsigned long arg);
>>
>> +#ifdef CONFIG_EEH
>> +extern int vfio_spapr_pci_eeh_open(struct pci_dev *pdev);
>> +extern void vfio_spapr_pci_eeh_release(struct pci_dev *pdev);
>> +extern long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group,
>> + unsigned int cmd,
>> + unsigned long arg);
>> +#else
>> +static inline int vfio_spapr_pci_eeh_open(struct pci_dev *pdev)
>> +{
>> + return 0;
>> +}
>> +
>> +static inline void vfio_spapr_pci_eeh_release(struct pci_dev *pdev)
>> +{
>> +}
>> +
>> +static inline long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group,
>> + unsigned int cmd,
>> + unsigned long arg)
>> +{
>> + return -ENOTTY;
>> +}
>> +#endif /* CONFIG_EEH */
>> #endif /* VFIO_H */
>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
>> index cb9023d..6612974 100644
>> --- a/include/uapi/linux/vfio.h
>> +++ b/include/uapi/linux/vfio.h
>> @@ -30,6 +30,9 @@
>> */
>> #define VFIO_DMA_CC_IOMMU 4
>>
>> +/* Check if EEH is supported */
>> +#define VFIO_EEH 5
>> +
>> /*
>> * The IOCTL interface is designed for extensibility by embedding the
>> * structure length (argsz) and flags into structures passed between
>> @@ -455,6 +458,37 @@ struct vfio_iommu_spapr_tce_info {
>>
>> #define VFIO_IOMMU_SPAPR_TCE_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
>>
>> +/*
>> + * EEH PE operation struct provides ways to:
>> + * - enable/disable EEH functionality;
>> + * - unfreeze IO/DMA for frozen PE;
>> + * - read PE state;
>> + * - reset PE;
>> + * - configure PE.
>> + */
>> +struct vfio_eeh_pe_op {
>> + __u32 argsz;
>> + __u32 flags;
>> + __u32 op;
>> +};
>> +
>> +#define VFIO_EEH_PE_DISABLE 0 /* Disable EEH functionality */
>> +#define VFIO_EEH_PE_ENABLE 1 /* Enable EEH functionality */
>> +#define VFIO_EEH_PE_UNFREEZE_IO 2 /* Enable IO for frozen PE */
>> +#define VFIO_EEH_PE_UNFREEZE_DMA 3 /* Enable DMA for frozen PE */
>> +#define VFIO_EEH_PE_GET_STATE 4 /* PE state retrieval */
>> +#define VFIO_EEH_PE_STATE_NORMAL 0 /* PE in functional state */
>> +#define VFIO_EEH_PE_STATE_RESET 1 /* PE reset in progress */
>> +#define VFIO_EEH_PE_STATE_STOPPED 2 /* Stopped DMA and IO */
>> +#define VFIO_EEH_PE_STATE_STOPPED_DMA 4 /* Stopped DMA only */
>> +#define VFIO_EEH_PE_STATE_UNAVAIL 5 /* State unavailable */
>> +#define VFIO_EEH_PE_RESET_DEACTIVATE 5 /* Deassert PE reset */
>> +#define VFIO_EEH_PE_RESET_HOT 6 /* Assert hot reset */
>> +#define VFIO_EEH_PE_RESET_FUNDAMENTAL 7 /* Assert fundamental reset */
>> +#define VFIO_EEH_PE_CONFIGURE 8 /* PE configuration */
>> +
>> +#define VFIO_EEH_PE_OP _IO(VFIO_TYPE, VFIO_BASE + 21)
>> +
>> /* ***************************************************************** */
>>
>> #endif /* _UAPIVFIO_H */
>
>
>
^ permalink raw reply
* Re: [PATCH] powerpc/powernv: fix endianness problems in EEH
From: Gavin Shan @ 2014-06-10 0:28 UTC (permalink / raw)
To: Guo Chao; +Cc: linuxppc-dev, Gavin Shan
In-Reply-To: <1402304331-24167-1-git-send-email-yan@linux.vnet.ibm.com>
On Mon, Jun 09, 2014 at 04:58:51PM +0800, Guo Chao wrote:
>EEH information fetched from OPAL need fix before using in LE environment.
>To be included in sparse's endian check, declare them as __beXX and
>access them by accessors.
>
>Cc: Gavin Shan <gwshan@linux.vnet.ibm.com>
>
>Signed-off-by: Guo Chao <yan@linux.vnet.ibm.com>
Acked-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
It looks good to me., but there are more diag-data needed to be
coverted: P7IOC diag-data and P7IOC PHB diag-data.
Thanks,
Gavin
>---
> arch/powerpc/include/asm/opal.h | 102 +++++++++++++++---------------
> arch/powerpc/platforms/powernv/eeh-ioda.c | 36 ++++++-----
> arch/powerpc/platforms/powernv/pci.c | 81 +++++++++++++++---------
> 3 files changed, 120 insertions(+), 99 deletions(-)
>
>diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h
>index 4ccd1d4..a4c0acc 100644
>--- a/arch/powerpc/include/asm/opal.h
>+++ b/arch/powerpc/include/asm/opal.h
>@@ -599,9 +599,9 @@ enum {
> };
>
> struct OpalIoPhbErrorCommon {
>- uint32_t version;
>- uint32_t ioType;
>- uint32_t len;
>+ __be32 version;
>+ __be32 ioType;
>+ __be32 len;
> };
>
> struct OpalIoP7IOCPhbErrorData {
>@@ -671,64 +671,64 @@ enum {
> struct OpalIoPhb3ErrorData {
> struct OpalIoPhbErrorCommon common;
>
>- uint32_t brdgCtl;
>+ __be32 brdgCtl;
>
> /* PHB3 UTL regs */
>- uint32_t portStatusReg;
>- uint32_t rootCmplxStatus;
>- uint32_t busAgentStatus;
>+ __be32 portStatusReg;
>+ __be32 rootCmplxStatus;
>+ __be32 busAgentStatus;
>
> /* PHB3 cfg regs */
>- uint32_t deviceStatus;
>- uint32_t slotStatus;
>- uint32_t linkStatus;
>- uint32_t devCmdStatus;
>- uint32_t devSecStatus;
>+ __be32 deviceStatus;
>+ __be32 slotStatus;
>+ __be32 linkStatus;
>+ __be32 devCmdStatus;
>+ __be32 devSecStatus;
>
> /* cfg AER regs */
>- uint32_t rootErrorStatus;
>- uint32_t uncorrErrorStatus;
>- uint32_t corrErrorStatus;
>- uint32_t tlpHdr1;
>- uint32_t tlpHdr2;
>- uint32_t tlpHdr3;
>- uint32_t tlpHdr4;
>- uint32_t sourceId;
>+ __be32 rootErrorStatus;
>+ __be32 uncorrErrorStatus;
>+ __be32 corrErrorStatus;
>+ __be32 tlpHdr1;
>+ __be32 tlpHdr2;
>+ __be32 tlpHdr3;
>+ __be32 tlpHdr4;
>+ __be32 sourceId;
>
>- uint32_t rsv3;
>+ __be32 rsv3;
>
> /* Record data about the call to allocate a buffer */
>- uint64_t errorClass;
>- uint64_t correlator;
>+ __be64 errorClass;
>+ __be64 correlator;
>
>- uint64_t nFir; /* 000 */
>- uint64_t nFirMask; /* 003 */
>- uint64_t nFirWOF; /* 008 */
>+ __be64 nFir; /* 000 */
>+ __be64 nFirMask; /* 003 */
>+ __be64 nFirWOF; /* 008 */
>
> /* PHB3 MMIO Error Regs */
>- uint64_t phbPlssr; /* 120 */
>- uint64_t phbCsr; /* 110 */
>- uint64_t lemFir; /* C00 */
>- uint64_t lemErrorMask; /* C18 */
>- uint64_t lemWOF; /* C40 */
>- uint64_t phbErrorStatus; /* C80 */
>- uint64_t phbFirstErrorStatus; /* C88 */
>- uint64_t phbErrorLog0; /* CC0 */
>- uint64_t phbErrorLog1; /* CC8 */
>- uint64_t mmioErrorStatus; /* D00 */
>- uint64_t mmioFirstErrorStatus; /* D08 */
>- uint64_t mmioErrorLog0; /* D40 */
>- uint64_t mmioErrorLog1; /* D48 */
>- uint64_t dma0ErrorStatus; /* D80 */
>- uint64_t dma0FirstErrorStatus; /* D88 */
>- uint64_t dma0ErrorLog0; /* DC0 */
>- uint64_t dma0ErrorLog1; /* DC8 */
>- uint64_t dma1ErrorStatus; /* E00 */
>- uint64_t dma1FirstErrorStatus; /* E08 */
>- uint64_t dma1ErrorLog0; /* E40 */
>- uint64_t dma1ErrorLog1; /* E48 */
>- uint64_t pestA[OPAL_PHB3_NUM_PEST_REGS];
>- uint64_t pestB[OPAL_PHB3_NUM_PEST_REGS];
>+ __be64 phbPlssr; /* 120 */
>+ __be64 phbCsr; /* 110 */
>+ __be64 lemFir; /* C00 */
>+ __be64 lemErrorMask; /* C18 */
>+ __be64 lemWOF; /* C40 */
>+ __be64 phbErrorStatus; /* C80 */
>+ __be64 phbFirstErrorStatus; /* C88 */
>+ __be64 phbErrorLog0; /* CC0 */
>+ __be64 phbErrorLog1; /* CC8 */
>+ __be64 mmioErrorStatus; /* D00 */
>+ __be64 mmioFirstErrorStatus; /* D08 */
>+ __be64 mmioErrorLog0; /* D40 */
>+ __be64 mmioErrorLog1; /* D48 */
>+ __be64 dma0ErrorStatus; /* D80 */
>+ __be64 dma0FirstErrorStatus; /* D88 */
>+ __be64 dma0ErrorLog0; /* DC0 */
>+ __be64 dma0ErrorLog1; /* DC8 */
>+ __be64 dma1ErrorStatus; /* E00 */
>+ __be64 dma1FirstErrorStatus; /* E08 */
>+ __be64 dma1ErrorLog0; /* E40 */
>+ __be64 dma1ErrorLog1; /* E48 */
>+ __be64 pestA[OPAL_PHB3_NUM_PEST_REGS];
>+ __be64 pestB[OPAL_PHB3_NUM_PEST_REGS];
> };
>
> typedef struct oppanel_line {
>@@ -851,8 +851,8 @@ int64_t opal_pci_mask_pe_error(uint64_t phb_id, uint16_t pe_number, uint8_t erro
> int64_t opal_set_slot_led_status(uint64_t phb_id, uint64_t slot_id, uint8_t led_type, uint8_t led_action);
> int64_t opal_get_epow_status(__be64 *status);
> int64_t opal_set_system_attention_led(uint8_t led_action);
>-int64_t opal_pci_next_error(uint64_t phb_id, uint64_t *first_frozen_pe,
>- uint16_t *pci_error_type, uint16_t *severity);
>+int64_t opal_pci_next_error(uint64_t phb_id, __be64 *first_frozen_pe,
>+ __be16 *pci_error_type, __be16 *severity);
> int64_t opal_pci_poll(uint64_t phb_id);
> int64_t opal_return_cpu(void);
> int64_t opal_reinit_cpus(uint64_t flags);
>diff --git a/arch/powerpc/platforms/powernv/eeh-ioda.c b/arch/powerpc/platforms/powernv/eeh-ioda.c
>index 753f08e..e0d6a3a 100644
>--- a/arch/powerpc/platforms/powernv/eeh-ioda.c
>+++ b/arch/powerpc/platforms/powernv/eeh-ioda.c
>@@ -267,7 +267,7 @@ static int ioda_eeh_get_state(struct eeh_pe *pe)
> {
> s64 ret = 0;
> u8 fstate;
>- u16 pcierr;
>+ __be16 pcierr;
> u32 pe_no;
> int result;
> struct pci_controller *hose = pe->phb;
>@@ -316,7 +316,7 @@ static int ioda_eeh_get_state(struct eeh_pe *pe)
> result = 0;
> result &= ~EEH_STATE_RESET_ACTIVE;
>
>- if (pcierr != OPAL_EEH_PHB_ERROR) {
>+ if (be16_to_cpu(pcierr) != OPAL_EEH_PHB_ERROR) {
> result |= EEH_STATE_MMIO_ACTIVE;
> result |= EEH_STATE_DMA_ACTIVE;
> result |= EEH_STATE_MMIO_ENABLED;
>@@ -706,8 +706,8 @@ static int ioda_eeh_next_error(struct eeh_pe **pe)
> struct pci_controller *hose;
> struct pnv_phb *phb;
> struct eeh_pe *phb_pe;
>- u64 frozen_pe_no;
>- u16 err_type, severity;
>+ __be64 frozen_pe_no;
>+ __be16 err_type, severity;
> long rc;
> int ret = EEH_NEXT_ERR_NONE;
>
>@@ -742,8 +742,8 @@ static int ioda_eeh_next_error(struct eeh_pe **pe)
> }
>
> /* If the PHB doesn't have error, stop processing */
>- if (err_type == OPAL_EEH_NO_ERROR ||
>- severity == OPAL_EEH_SEV_NO_ERROR) {
>+ if (be16_to_cpu(err_type) == OPAL_EEH_NO_ERROR ||
>+ be16_to_cpu(severity) == OPAL_EEH_SEV_NO_ERROR) {
> pr_devel("%s: No error found on PHB#%x\n",
> __func__, hose->global_number);
> continue;
>@@ -755,14 +755,14 @@ static int ioda_eeh_next_error(struct eeh_pe **pe)
> * specific PHB.
> */
> pr_devel("%s: Error (%d, %d, %llu) on PHB#%x\n",
>- __func__, err_type, severity,
>- frozen_pe_no, hose->global_number);
>- switch (err_type) {
>+ __func__, be16_to_cpu(err_type), be16_to_cpu(severity),
>+ be64_to_cpu(frozen_pe_no), hose->global_number);
>+ switch (be16_to_cpu(err_type)) {
> case OPAL_EEH_IOC_ERROR:
>- if (severity == OPAL_EEH_SEV_IOC_DEAD) {
>+ if (be16_to_cpu(severity) == OPAL_EEH_SEV_IOC_DEAD) {
> pr_err("EEH: dead IOC detected\n");
> ret = EEH_NEXT_ERR_DEAD_IOC;
>- } else if (severity == OPAL_EEH_SEV_INF) {
>+ } else if (be16_to_cpu(severity) == OPAL_EEH_SEV_INF) {
> pr_info("EEH: IOC informative error "
> "detected\n");
> ioda_eeh_hub_diag(hose);
>@@ -771,17 +771,18 @@ static int ioda_eeh_next_error(struct eeh_pe **pe)
>
> break;
> case OPAL_EEH_PHB_ERROR:
>- if (severity == OPAL_EEH_SEV_PHB_DEAD) {
>+ if (be16_to_cpu(severity) == OPAL_EEH_SEV_PHB_DEAD) {
> *pe = phb_pe;
> pr_err("EEH: dead PHB#%x detected\n",
> hose->global_number);
> ret = EEH_NEXT_ERR_DEAD_PHB;
>- } else if (severity == OPAL_EEH_SEV_PHB_FENCED) {
>+ } else if (be16_to_cpu(severity) ==
>+ OPAL_EEH_SEV_PHB_FENCED) {
> *pe = phb_pe;
> pr_err("EEH: fenced PHB#%x detected\n",
> hose->global_number);
> ret = EEH_NEXT_ERR_FENCED_PHB;
>- } else if (severity == OPAL_EEH_SEV_INF) {
>+ } else if (be16_to_cpu(severity) == OPAL_EEH_SEV_INF) {
> pr_info("EEH: PHB#%x informative error "
> "detected\n",
> hose->global_number);
>@@ -801,12 +802,13 @@ static int ioda_eeh_next_error(struct eeh_pe **pe)
> * progress with recovery. We needn't report
> * it again.
> */
>- if (ioda_eeh_get_pe(hose, frozen_pe_no, pe)) {
>+ if (ioda_eeh_get_pe(hose,
>+ be64_to_cpu(frozen_pe_no), pe)) {
> *pe = phb_pe;
> pr_err("EEH: Escalated fenced PHB#%x "
> "detected for PE#%llx\n",
> hose->global_number,
>- frozen_pe_no);
>+ be64_to_cpu(frozen_pe_no));
> ret = EEH_NEXT_ERR_FENCED_PHB;
> } else if ((*pe)->state & EEH_PE_ISOLATED) {
> ret = EEH_NEXT_ERR_NONE;
>@@ -819,7 +821,7 @@ static int ioda_eeh_next_error(struct eeh_pe **pe)
> break;
> default:
> pr_warn("%s: Unexpected error type %d\n",
>- __func__, err_type);
>+ __func__, be16_to_cpu(err_type));
> }
>
> /*
>diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c
>index eefbfcc..f91a4e5 100644
>--- a/arch/powerpc/platforms/powernv/pci.c
>+++ b/arch/powerpc/platforms/powernv/pci.c
>@@ -206,72 +206,91 @@ static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
>
> data = (struct OpalIoPhb3ErrorData*)common;
> pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
>- hose->global_number, common->version);
>+ hose->global_number, be32_to_cpu(common->version));
> if (data->brdgCtl)
> pr_info("brdgCtl: %08x\n",
>- data->brdgCtl);
>+ be32_to_cpu(data->brdgCtl));
> if (data->portStatusReg || data->rootCmplxStatus ||
> data->busAgentStatus)
> pr_info("UtlSts: %08x %08x %08x\n",
>- data->portStatusReg, data->rootCmplxStatus,
>- data->busAgentStatus);
>+ be32_to_cpu(data->portStatusReg),
>+ be32_to_cpu(data->rootCmplxStatus),
>+ be32_to_cpu(data->busAgentStatus));
> if (data->deviceStatus || data->slotStatus ||
> data->linkStatus || data->devCmdStatus ||
> data->devSecStatus)
> pr_info("RootSts: %08x %08x %08x %08x %08x\n",
>- data->deviceStatus, data->slotStatus,
>- data->linkStatus, data->devCmdStatus,
>- data->devSecStatus);
>+ be32_to_cpu(data->deviceStatus),
>+ be32_to_cpu(data->slotStatus),
>+ be32_to_cpu(data->linkStatus),
>+ be32_to_cpu(data->devCmdStatus),
>+ be32_to_cpu(data->devSecStatus));
> if (data->rootErrorStatus || data->uncorrErrorStatus ||
> data->corrErrorStatus)
> pr_info("RootErrSts: %08x %08x %08x\n",
>- data->rootErrorStatus, data->uncorrErrorStatus,
>- data->corrErrorStatus);
>+ be32_to_cpu(data->rootErrorStatus),
>+ be32_to_cpu(data->uncorrErrorStatus),
>+ be32_to_cpu(data->corrErrorStatus));
> if (data->tlpHdr1 || data->tlpHdr2 ||
> data->tlpHdr3 || data->tlpHdr4)
> pr_info("RootErrLog: %08x %08x %08x %08x\n",
>- data->tlpHdr1, data->tlpHdr2,
>- data->tlpHdr3, data->tlpHdr4);
>+ be32_to_cpu(data->tlpHdr1),
>+ be32_to_cpu(data->tlpHdr2),
>+ be32_to_cpu(data->tlpHdr3),
>+ be32_to_cpu(data->tlpHdr4));
> if (data->sourceId || data->errorClass ||
> data->correlator)
> pr_info("RootErrLog1: %08x %016llx %016llx\n",
>- data->sourceId, data->errorClass,
>- data->correlator);
>+ be32_to_cpu(data->sourceId),
>+ be64_to_cpu(data->errorClass),
>+ be64_to_cpu(data->correlator));
> if (data->nFir)
> pr_info("nFir: %016llx %016llx %016llx\n",
>- data->nFir, data->nFirMask,
>- data->nFirWOF);
>+ be64_to_cpu(data->nFir),
>+ be64_to_cpu(data->nFirMask),
>+ be64_to_cpu(data->nFirWOF));
> if (data->phbPlssr || data->phbCsr)
> pr_info("PhbSts: %016llx %016llx\n",
>- data->phbPlssr, data->phbCsr);
>+ be64_to_cpu(data->phbPlssr),
>+ be64_to_cpu(data->phbCsr));
> if (data->lemFir)
> pr_info("Lem: %016llx %016llx %016llx\n",
>- data->lemFir, data->lemErrorMask,
>- data->lemWOF);
>+ be64_to_cpu(data->lemFir),
>+ be64_to_cpu(data->lemErrorMask),
>+ be64_to_cpu(data->lemWOF));
> if (data->phbErrorStatus)
> pr_info("PhbErr: %016llx %016llx %016llx %016llx\n",
>- data->phbErrorStatus, data->phbFirstErrorStatus,
>- data->phbErrorLog0, data->phbErrorLog1);
>+ be64_to_cpu(data->phbErrorStatus),
>+ be64_to_cpu(data->phbFirstErrorStatus),
>+ be64_to_cpu(data->phbErrorLog0),
>+ be64_to_cpu(data->phbErrorLog1));
> if (data->mmioErrorStatus)
> pr_info("OutErr: %016llx %016llx %016llx %016llx\n",
>- data->mmioErrorStatus, data->mmioFirstErrorStatus,
>- data->mmioErrorLog0, data->mmioErrorLog1);
>+ be64_to_cpu(data->mmioErrorStatus),
>+ be64_to_cpu(data->mmioFirstErrorStatus),
>+ be64_to_cpu(data->mmioErrorLog0),
>+ be64_to_cpu(data->mmioErrorLog1));
> if (data->dma0ErrorStatus)
> pr_info("InAErr: %016llx %016llx %016llx %016llx\n",
>- data->dma0ErrorStatus, data->dma0FirstErrorStatus,
>- data->dma0ErrorLog0, data->dma0ErrorLog1);
>+ be64_to_cpu(data->dma0ErrorStatus),
>+ be64_to_cpu(data->dma0FirstErrorStatus),
>+ be64_to_cpu(data->dma0ErrorLog0),
>+ be64_to_cpu(data->dma0ErrorLog1));
> if (data->dma1ErrorStatus)
> pr_info("InBErr: %016llx %016llx %016llx %016llx\n",
>- data->dma1ErrorStatus, data->dma1FirstErrorStatus,
>- data->dma1ErrorLog0, data->dma1ErrorLog1);
>+ be64_to_cpu(data->dma1ErrorStatus),
>+ be64_to_cpu(data->dma1FirstErrorStatus),
>+ be64_to_cpu(data->dma1ErrorLog0),
>+ be64_to_cpu(data->dma1ErrorLog1));
>
> for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
>- if ((data->pestA[i] >> 63) == 0 &&
>- (data->pestB[i] >> 63) == 0)
>+ if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
>+ (be64_to_cpu(data->pestB[i]) >> 63) == 0)
> continue;
>
> pr_info("PE[%3d] A/B: %016llx %016llx\n",
>- i, data->pestA[i], data->pestB[i]);
>+ i, be64_to_cpu(data->pestA[i]),
>+ be64_to_cpu(data->pestB[i]));
> }
> }
>
>@@ -284,7 +303,7 @@ void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
> return;
>
> common = (struct OpalIoPhbErrorCommon *)log_buff;
>- switch (common->ioType) {
>+ switch (be32_to_cpu(common->ioType)) {
> case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
> pnv_pci_dump_p7ioc_diag_data(hose, common);
> break;
>@@ -293,7 +312,7 @@ void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
> break;
> default:
> pr_warn("%s: Unrecognized ioType %d\n",
>- __func__, common->ioType);
>+ __func__, be32_to_cpu(common->ioType));
> }
> }
>
>--
>1.9.1
>
^ permalink raw reply
* [PATCH v10 0/3] EEH Support for VFIO PCI Device
From: Gavin Shan @ 2014-06-10 1:41 UTC (permalink / raw)
To: kvm-ppc, linuxppc-dev; +Cc: aik, agraf, Gavin Shan, alex.williamson, qiudayu
EEH Support for VFIO PCI Device
The series of patches adds support EEH for PCI devices, which are passed
through to PowerKVM based guest via VFIO. The implementation is straightforward
based on the issues or problems we have to resolve to support EEH for PowerKVM
based guest.
- Emulation for EEH RTAS requests. All EEH RTAS requests goes to QEMU firstly.
If QEMU can't handle it, the request will be sent to host via newly introduced
VFIO container IOCTL command (VFIO_EEH_OP) and gets handled in host kernel.
The series of patches requires corresponding QEMU changes.
Change log
==========
v1 -> v2:
* EEH RTAS requests are routed to QEMU, and then possiblly to host kerenl.
The mechanism KVM in-kernel handling is dropped.
* Error injection is reimplemented based syscall, instead of KVM in-kerenl
handling. The logic for error injection token management is moved to
QEMU. The error injection request is routed to QEMU and then possiblly
to host kernel.
v2 -> v3:
* Make the fields in struct eeh_vfio_pci_addr, struct vfio_eeh_info based
on the comments from Alexey.
* Define macros for EEH VFIO operations (Alexey).
* Clear frozen state after successful PE reset.
* Merge original [PATCH 1/2/3] to one.
v3 -> v4:
* Remove the error injection from the patchset. Mike or I will work on that
later.
* Rename CONFIG_VFIO_EEH to VFIO_PCI_EEH.
* Rename the IOCTL command to VFIO_EEH_OP and it's handled by VFIO-PCI device
instead of VFIO container.
* Rename the IOCTL argument structure to "vfio_eeh_op" accordingly. Also, more
fields added to hold return values for RTAS requests.
* The address mapping stuff is totally removed. When opening or releasing VFIO
PCI device, notification sent to EEH to update the flags indicates the device
is passed to guest or not.
* Change pr_warn() to pr_debug() to avoid DOS as pointed by Alex.W
* Argument size check issue pointed by Alex.W.
v4 -> v5:
* Functions for VFIO PCI EEH support are moved to eeh.c and exported from there.
VFIO PCI driver just uses those functions to tackle IOCTL command VFIO_EEH_OP.
All of this is to make the code organized in a good way as suggested by Alex.G.
Another potential benefit is PowerNV/pSeries are sharing "eeh_ops" and same
infrastructure could possiblly work for KVM_PR and KVM_HV mode at the same time.
* Don't clear error injection registers after finishing PE reset as the patchset
is doing nothing related to error injection.
* Amending Documentation/vfio.txt, which was missed in last revision.
* No QEMU changes for this revision. "v4" works well. Also, remove "RFC" from the
subject as the design is basically recognized.
v5 -> v6:
* CONFIG_VFIO_PCI_EEH removed. Instead to use CONFIG_EEH.
* Split one ioctl command to 5.
* In eeh.c, description has been added for those exported functions. Also, the
functions have negative return values for error and information with other values.
All digital numbers have been replaced by macros defined in eeh.h. The comments,
including the function names have been amended not to mention "guest" or "vfio".
* Add one mutex to protect flag in eeh_dev_open()/release().
* More information on how to use those ioctl commands to Documentation/vfio.txt.
v6 -> v7:
* Remove ioctl command VFIO_EEH_PE_GET_ADDR, the PE address will be figured out
in userland (e.g. QEMU) as Alex.G suggested.
* Let sPAPR VFIO container process the ioctl commands as VFIO container is naturally
corresponds to IOMMU group (aka PE on sPAPR platform).
* All VFIO PCI EEH ioctl commands have "argsz+flags" for its companion data struct.
* For VFIO PCI EEH ioctl commands, ioctl() returns negative number to indicate error
or zero for success. Additinal output information is transported by the companion
data struct.
* Explaining PE in Documentation/vfio.txt, typo fixes, more comments suggested by
Alex.G.
* Split/merge patches according to suggestions from Alex.G and Alex.W.
* To have EEH stub in drivers/vfio/pci/, which was suggested by Alex.W.
* Define various EEH options as macros in vfio.h for userland to use.
v7 -> v8:
* Change ioctl commands back to combined one.
* EEH related logic was put into drivers/vfio/vfio_eeh.c, which is only built with
CONFIG_EEH. Otherwise, inline functions defined in include/linux/vfio.h
* Change vfio.txt according to the source code changes.
* Fix various comments from internal reviews by Alexey. Thanks to Alexey.
v8 -> v9:
* Remove unused macros in asm/include/eeh.h
* Missed to disable VFIO device on error from vfio_spapr_pci_eeh_open().
* Don't include unused header files in drivers/vfio/vfio_spapr_eeh.c
* Define inline PE state for VFIO_EEH_PE_GET_STATE.
v9 -> v10:
* Make sure we have zero struct vfio_eeh_pe_op::flags
Gavin Shan (3):
powerpc/eeh: Avoid event on passed PE
powerpc/eeh: EEH support for VFIO PCI device
drivers/vfio: EEH support for VFIO PCI device
Documentation/vfio.txt | 87 +++++++++-
arch/powerpc/include/asm/eeh.h | 19 ++
arch/powerpc/kernel/eeh.c | 276 ++++++++++++++++++++++++++++++
arch/powerpc/platforms/powernv/eeh-ioda.c | 3 +-
drivers/vfio/Makefile | 1 +
drivers/vfio/pci/vfio_pci.c | 18 +-
drivers/vfio/vfio_iommu_spapr_tce.c | 17 +-
drivers/vfio/vfio_spapr_eeh.c | 87 ++++++++++
include/linux/vfio.h | 23 +++
include/uapi/linux/vfio.h | 34 ++++
10 files changed, 556 insertions(+), 9 deletions(-)
create mode 100644 drivers/vfio/vfio_spapr_eeh.c
--
1.8.3.2
^ permalink raw reply
* [PATCH v10 1/3] powerpc/eeh: Avoid event on passed PE
From: Gavin Shan @ 2014-06-10 1:41 UTC (permalink / raw)
To: kvm-ppc, linuxppc-dev; +Cc: aik, agraf, Gavin Shan, alex.williamson, qiudayu
In-Reply-To: <1402364517-28561-1-git-send-email-gwshan@linux.vnet.ibm.com>
We must not handle EEH error on devices which are passed to somebody
else. Instead, we expect that the frozen device owner detects an EEH
error and recovers from it.
This avoids EEH error handling on passed through devices so the device
owner gets a chance to handle them.
Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
Acked-by: Alexander Graf <agraf@suse.de>
---
arch/powerpc/include/asm/eeh.h | 7 +++++++
arch/powerpc/kernel/eeh.c | 8 ++++++++
arch/powerpc/platforms/powernv/eeh-ioda.c | 3 ++-
3 files changed, 17 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h
index 7782056..653d981 100644
--- a/arch/powerpc/include/asm/eeh.h
+++ b/arch/powerpc/include/asm/eeh.h
@@ -25,6 +25,7 @@
#include <linux/list.h>
#include <linux/string.h>
#include <linux/time.h>
+#include <linux/atomic.h>
struct pci_dev;
struct pci_bus;
@@ -84,6 +85,7 @@ struct eeh_pe {
int freeze_count; /* Times of froze up */
struct timeval tstamp; /* Time on first-time freeze */
int false_positives; /* Times of reported #ff's */
+ atomic_t pass_dev_cnt; /* Count of passed through devs */
struct eeh_pe *parent; /* Parent PE */
struct list_head child_list; /* Link PE to the child list */
struct list_head edevs; /* Link list of EEH devices */
@@ -93,6 +95,11 @@ struct eeh_pe {
#define eeh_pe_for_each_dev(pe, edev, tmp) \
list_for_each_entry_safe(edev, tmp, &pe->edevs, list)
+static inline bool eeh_pe_passed(struct eeh_pe *pe)
+{
+ return pe ? !!atomic_read(&pe->pass_dev_cnt) : false;
+}
+
/*
* The struct is used to trace EEH state for the associated
* PCI device node or PCI device. In future, it might
diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
index 9c6b899..3bc8b12 100644
--- a/arch/powerpc/kernel/eeh.c
+++ b/arch/powerpc/kernel/eeh.c
@@ -400,6 +400,14 @@ int eeh_dev_check_failure(struct eeh_dev *edev)
if (ret > 0)
return ret;
+ /*
+ * If the PE isn't owned by us, we shouldn't check the
+ * state. Instead, let the owner handle it if the PE has
+ * been frozen.
+ */
+ if (eeh_pe_passed(pe))
+ return 0;
+
/* If we already have a pending isolation event for this
* slot, we know it's bad already, we don't need to check.
* Do this checking under a lock; as multiple PCI devices
diff --git a/arch/powerpc/platforms/powernv/eeh-ioda.c b/arch/powerpc/platforms/powernv/eeh-ioda.c
index cab3e62..79193eb 100644
--- a/arch/powerpc/platforms/powernv/eeh-ioda.c
+++ b/arch/powerpc/platforms/powernv/eeh-ioda.c
@@ -892,7 +892,8 @@ static int ioda_eeh_next_error(struct eeh_pe **pe)
opal_pci_eeh_freeze_clear(phb->opal_id, frozen_pe_no,
OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
ret = EEH_NEXT_ERR_NONE;
- } else if ((*pe)->state & EEH_PE_ISOLATED) {
+ } else if ((*pe)->state & EEH_PE_ISOLATED ||
+ eeh_pe_passed(*pe)) {
ret = EEH_NEXT_ERR_NONE;
} else {
pr_err("EEH: Frozen PHB#%x-PE#%x (%s) detected\n",
--
1.8.3.2
^ permalink raw reply related
* [PATCH v10 3/3] drivers/vfio: EEH support for VFIO PCI device
From: Gavin Shan @ 2014-06-10 1:41 UTC (permalink / raw)
To: kvm-ppc, linuxppc-dev; +Cc: aik, agraf, Gavin Shan, alex.williamson, qiudayu
In-Reply-To: <1402364517-28561-1-git-send-email-gwshan@linux.vnet.ibm.com>
The patch adds new IOCTL commands for sPAPR VFIO container device
to support EEH functionality for PCI devices, which have been passed
through from host to somebody else via VFIO.
Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
Acked-by: Alexander Graf <agraf@suse.de>
---
Documentation/vfio.txt | 87 +++++++++++++++++++++++++++++++++++--
drivers/vfio/Makefile | 1 +
drivers/vfio/pci/vfio_pci.c | 18 ++++++--
drivers/vfio/vfio_iommu_spapr_tce.c | 17 +++++++-
drivers/vfio/vfio_spapr_eeh.c | 87 +++++++++++++++++++++++++++++++++++++
include/linux/vfio.h | 23 ++++++++++
include/uapi/linux/vfio.h | 34 +++++++++++++++
7 files changed, 259 insertions(+), 8 deletions(-)
create mode 100644 drivers/vfio/vfio_spapr_eeh.c
diff --git a/Documentation/vfio.txt b/Documentation/vfio.txt
index b9ca023..96978ec 100644
--- a/Documentation/vfio.txt
+++ b/Documentation/vfio.txt
@@ -305,7 +305,15 @@ faster, the map/unmap handling has been implemented in real mode which provides
an excellent performance which has limitations such as inability to do
locked pages accounting in real time.
-So 3 additional ioctls have been added:
+4) According to sPAPR specification, A Partitionable Endpoint (PE) is an I/O
+subtree that can be treated as a unit for the purposes of partitioning and
+error recovery. A PE may be a single or multi-function IOA (IO Adapter), a
+function of a multi-function IOA, or multiple IOAs (possibly including switch
+and bridge structures above the multiple IOAs). PPC64 guests detect PCI errors
+and recover from them via EEH RTAS services, which works on the basis of
+additional ioctl commands.
+
+So 4 additional ioctls have been added:
VFIO_IOMMU_SPAPR_TCE_GET_INFO - returns the size and the start
of the DMA window on the PCI bus.
@@ -316,9 +324,12 @@ So 3 additional ioctls have been added:
VFIO_IOMMU_DISABLE - disables the container.
+ VFIO_EEH_PE_OP - provides an API for EEH setup, error detection and recovery.
The code flow from the example above should be slightly changed:
+ struct vfio_eeh_pe_op pe_op = { .argsz = sizeof(pe_op), .flags = 0 };
+
.....
/* Add the group to the container */
ioctl(group, VFIO_GROUP_SET_CONTAINER, &container);
@@ -342,9 +353,79 @@ The code flow from the example above should be slightly changed:
dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
/* Check here is .iova/.size are within DMA window from spapr_iommu_info */
-
ioctl(container, VFIO_IOMMU_MAP_DMA, &dma_map);
- .....
+
+ /* Get a file descriptor for the device */
+ device = ioctl(group, VFIO_GROUP_GET_DEVICE_FD, "0000:06:0d.0");
+
+ ....
+
+ /* Gratuitous device reset and go... */
+ ioctl(device, VFIO_DEVICE_RESET);
+
+ /* Make sure EEH is supported */
+ ioctl(container, VFIO_CHECK_EXTENSION, VFIO_EEH);
+
+ /* Enable the EEH functionality on the device */
+ pe_op.op = VFIO_EEH_PE_ENABLE;
+ ioctl(container, VFIO_EEH_PE_OP, &pe_op);
+
+ /* You're suggested to create additional data struct to represent
+ * PE, and put child devices belonging to same IOMMU group to the
+ * PE instance for later reference.
+ */
+
+ /* Check the PE's state and make sure it's in functional state */
+ pe_op.op = VFIO_EEH_PE_GET_STATE;
+ ioctl(container, VFIO_EEH_PE_OP, &pe_op);
+
+ /* Save device state using pci_save_state().
+ * EEH should be enabled on the specified device.
+ */
+
+ ....
+
+ /* When 0xFF's returned from reading PCI config space or IO BARs
+ * of the PCI device. Check the PE's state to see if that has been
+ * frozen.
+ */
+ ioctl(container, VFIO_EEH_PE_OP, &pe_op);
+
+ /* Waiting for pending PCI transactions to be completed and don't
+ * produce any more PCI traffic from/to the affected PE until
+ * recovery is finished.
+ */
+
+ /* Enable IO for the affected PE and collect logs. Usually, the
+ * standard part of PCI config space, AER registers are dumped
+ * as logs for further analysis.
+ */
+ pe_op.op = VFIO_EEH_PE_UNFREEZE_IO;
+ ioctl(container, VFIO_EEH_PE_OP, &pe_op);
+
+ /*
+ * Issue PE reset: hot or fundamental reset. Usually, hot reset
+ * is enough. However, the firmware of some PCI adapters would
+ * require fundamental reset.
+ */
+ pe_op.op = VFIO_EEH_PE_RESET_HOT;
+ ioctl(container, VFIO_EEH_PE_OP, &pe_op);
+ pe_op.op = VFIO_EEH_PE_RESET_DEACTIVATE;
+ ioctl(container, VFIO_EEH_PE_OP, &pe_op);
+
+ /* Configure the PCI bridges for the affected PE */
+ pe_op.op = VFIO_EEH_PE_CONFIGURE;
+ ioctl(container, VFIO_EEH_PE_OP, &pe_op);
+
+ /* Restored state we saved at initialization time. pci_restore_state()
+ * is good enough as an example.
+ */
+
+ /* Hopefully, error is recovered successfully. Now, you can resume to
+ * start PCI traffic to/from the affected PE.
+ */
+
+ ....
-------------------------------------------------------------------------------
diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
index 72bfabc..50e30bc 100644
--- a/drivers/vfio/Makefile
+++ b/drivers/vfio/Makefile
@@ -1,4 +1,5 @@
obj-$(CONFIG_VFIO) += vfio.o
obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
+obj-$(CONFIG_EEH) += vfio_spapr_eeh.o
obj-$(CONFIG_VFIO_PCI) += pci/
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index 7ba0424..0122665 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -156,8 +156,10 @@ static void vfio_pci_release(void *device_data)
{
struct vfio_pci_device *vdev = device_data;
- if (atomic_dec_and_test(&vdev->refcnt))
+ if (atomic_dec_and_test(&vdev->refcnt)) {
+ vfio_spapr_pci_eeh_release(vdev->pdev);
vfio_pci_disable(vdev);
+ }
module_put(THIS_MODULE);
}
@@ -165,19 +167,27 @@ static void vfio_pci_release(void *device_data)
static int vfio_pci_open(void *device_data)
{
struct vfio_pci_device *vdev = device_data;
+ int ret;
if (!try_module_get(THIS_MODULE))
return -ENODEV;
if (atomic_inc_return(&vdev->refcnt) == 1) {
- int ret = vfio_pci_enable(vdev);
+ ret = vfio_pci_enable(vdev);
+ if (ret)
+ goto error;
+
+ ret = vfio_spapr_pci_eeh_open(vdev->pdev);
if (ret) {
- module_put(THIS_MODULE);
- return ret;
+ vfio_pci_disable(vdev);
+ goto error;
}
}
return 0;
+error:
+ module_put(THIS_MODULE);
+ return ret;
}
static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c
index a84788b..730b4ef 100644
--- a/drivers/vfio/vfio_iommu_spapr_tce.c
+++ b/drivers/vfio/vfio_iommu_spapr_tce.c
@@ -156,7 +156,16 @@ static long tce_iommu_ioctl(void *iommu_data,
switch (cmd) {
case VFIO_CHECK_EXTENSION:
- return (arg == VFIO_SPAPR_TCE_IOMMU) ? 1 : 0;
+ switch (arg) {
+ case VFIO_SPAPR_TCE_IOMMU:
+ ret = 1;
+ break;
+ default:
+ ret = vfio_spapr_iommu_eeh_ioctl(NULL, cmd, arg);
+ break;
+ }
+
+ return (ret < 0) ? 0 : ret;
case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
struct vfio_iommu_spapr_tce_info info;
@@ -283,6 +292,12 @@ static long tce_iommu_ioctl(void *iommu_data,
tce_iommu_disable(container);
mutex_unlock(&container->lock);
return 0;
+ case VFIO_EEH_PE_OP:
+ if (!container->tbl || !container->tbl->it_group)
+ return -ENODEV;
+
+ return vfio_spapr_iommu_eeh_ioctl(container->tbl->it_group,
+ cmd, arg);
}
return -ENOTTY;
diff --git a/drivers/vfio/vfio_spapr_eeh.c b/drivers/vfio/vfio_spapr_eeh.c
new file mode 100644
index 0000000..f834b4c
--- /dev/null
+++ b/drivers/vfio/vfio_spapr_eeh.c
@@ -0,0 +1,87 @@
+/*
+ * EEH functionality support for VFIO devices. The feature is only
+ * available on sPAPR compatible platforms.
+ *
+ * Copyright Gavin Shan, IBM Corporation 2014.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/uaccess.h>
+#include <linux/vfio.h>
+#include <asm/eeh.h>
+
+/* We might build address mapping here for "fast" path later */
+int vfio_spapr_pci_eeh_open(struct pci_dev *pdev)
+{
+ return eeh_dev_open(pdev);
+}
+
+void vfio_spapr_pci_eeh_release(struct pci_dev *pdev)
+{
+ eeh_dev_release(pdev);
+}
+
+long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group,
+ unsigned int cmd, unsigned long arg)
+{
+ struct eeh_pe *pe;
+ struct vfio_eeh_pe_op op;
+ unsigned long minsz;
+ long ret = -EINVAL;
+
+ switch (cmd) {
+ case VFIO_CHECK_EXTENSION:
+ if (arg == VFIO_EEH)
+ ret = eeh_enabled() ? 1 : 0;
+ else
+ ret = 0;
+ break;
+ case VFIO_EEH_PE_OP:
+ pe = eeh_iommu_group_to_pe(group);
+ if (!pe)
+ return -ENODEV;
+
+ minsz = offsetofend(struct vfio_eeh_pe_op, op);
+ if (copy_from_user(&op, (void __user *)arg, minsz))
+ return -EFAULT;
+ if (op.argsz < minsz || op.flags)
+ return -EINVAL;
+
+ switch (op.op) {
+ case VFIO_EEH_PE_DISABLE:
+ ret = eeh_pe_set_option(pe, EEH_OPT_DISABLE);
+ break;
+ case VFIO_EEH_PE_ENABLE:
+ ret = eeh_pe_set_option(pe, EEH_OPT_ENABLE);
+ break;
+ case VFIO_EEH_PE_UNFREEZE_IO:
+ ret = eeh_pe_set_option(pe, EEH_OPT_THAW_MMIO);
+ break;
+ case VFIO_EEH_PE_UNFREEZE_DMA:
+ ret = eeh_pe_set_option(pe, EEH_OPT_THAW_DMA);
+ break;
+ case VFIO_EEH_PE_GET_STATE:
+ ret = eeh_pe_get_state(pe);
+ break;
+ case VFIO_EEH_PE_RESET_DEACTIVATE:
+ ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE);
+ break;
+ case VFIO_EEH_PE_RESET_HOT:
+ ret = eeh_pe_reset(pe, EEH_RESET_HOT);
+ break;
+ case VFIO_EEH_PE_RESET_FUNDAMENTAL:
+ ret = eeh_pe_reset(pe, EEH_RESET_FUNDAMENTAL);
+ break;
+ case VFIO_EEH_PE_CONFIGURE:
+ ret = eeh_pe_configure(pe);
+ break;
+ default:
+ ret = -EINVAL;
+ }
+ }
+
+ return ret;
+}
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index 81022a52..0d3bb8f 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -99,4 +99,27 @@ extern int vfio_external_user_iommu_id(struct vfio_group *group);
extern long vfio_external_check_extension(struct vfio_group *group,
unsigned long arg);
+#ifdef CONFIG_EEH
+extern int vfio_spapr_pci_eeh_open(struct pci_dev *pdev);
+extern void vfio_spapr_pci_eeh_release(struct pci_dev *pdev);
+extern long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group,
+ unsigned int cmd,
+ unsigned long arg);
+#else
+static inline int vfio_spapr_pci_eeh_open(struct pci_dev *pdev)
+{
+ return 0;
+}
+
+static inline void vfio_spapr_pci_eeh_release(struct pci_dev *pdev)
+{
+}
+
+static inline long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group,
+ unsigned int cmd,
+ unsigned long arg)
+{
+ return -ENOTTY;
+}
+#endif /* CONFIG_EEH */
#endif /* VFIO_H */
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index cb9023d..6612974 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -30,6 +30,9 @@
*/
#define VFIO_DMA_CC_IOMMU 4
+/* Check if EEH is supported */
+#define VFIO_EEH 5
+
/*
* The IOCTL interface is designed for extensibility by embedding the
* structure length (argsz) and flags into structures passed between
@@ -455,6 +458,37 @@ struct vfio_iommu_spapr_tce_info {
#define VFIO_IOMMU_SPAPR_TCE_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
+/*
+ * EEH PE operation struct provides ways to:
+ * - enable/disable EEH functionality;
+ * - unfreeze IO/DMA for frozen PE;
+ * - read PE state;
+ * - reset PE;
+ * - configure PE.
+ */
+struct vfio_eeh_pe_op {
+ __u32 argsz;
+ __u32 flags;
+ __u32 op;
+};
+
+#define VFIO_EEH_PE_DISABLE 0 /* Disable EEH functionality */
+#define VFIO_EEH_PE_ENABLE 1 /* Enable EEH functionality */
+#define VFIO_EEH_PE_UNFREEZE_IO 2 /* Enable IO for frozen PE */
+#define VFIO_EEH_PE_UNFREEZE_DMA 3 /* Enable DMA for frozen PE */
+#define VFIO_EEH_PE_GET_STATE 4 /* PE state retrieval */
+#define VFIO_EEH_PE_STATE_NORMAL 0 /* PE in functional state */
+#define VFIO_EEH_PE_STATE_RESET 1 /* PE reset in progress */
+#define VFIO_EEH_PE_STATE_STOPPED 2 /* Stopped DMA and IO */
+#define VFIO_EEH_PE_STATE_STOPPED_DMA 4 /* Stopped DMA only */
+#define VFIO_EEH_PE_STATE_UNAVAIL 5 /* State unavailable */
+#define VFIO_EEH_PE_RESET_DEACTIVATE 5 /* Deassert PE reset */
+#define VFIO_EEH_PE_RESET_HOT 6 /* Assert hot reset */
+#define VFIO_EEH_PE_RESET_FUNDAMENTAL 7 /* Assert fundamental reset */
+#define VFIO_EEH_PE_CONFIGURE 8 /* PE configuration */
+
+#define VFIO_EEH_PE_OP _IO(VFIO_TYPE, VFIO_BASE + 21)
+
/* ***************************************************************** */
#endif /* _UAPIVFIO_H */
--
1.8.3.2
^ permalink raw reply related
* [PATCH v10 2/3] powerpc/eeh: EEH support for VFIO PCI device
From: Gavin Shan @ 2014-06-10 1:41 UTC (permalink / raw)
To: kvm-ppc, linuxppc-dev; +Cc: aik, agraf, Gavin Shan, alex.williamson, qiudayu
In-Reply-To: <1402364517-28561-1-git-send-email-gwshan@linux.vnet.ibm.com>
The patch exports functions to be used by new VFIO ioctl command,
which will be introduced in subsequent patch, to support EEH
functinality for VFIO PCI devices.
Signed-off-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
Acked-by: Alexander Graf <agraf@suse.de>
---
arch/powerpc/include/asm/eeh.h | 12 ++
arch/powerpc/kernel/eeh.c | 268 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 280 insertions(+)
diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h
index 653d981..b733044 100644
--- a/arch/powerpc/include/asm/eeh.h
+++ b/arch/powerpc/include/asm/eeh.h
@@ -173,6 +173,11 @@ enum {
#define EEH_STATE_DMA_ACTIVE (1 << 4) /* Active DMA */
#define EEH_STATE_MMIO_ENABLED (1 << 5) /* MMIO enabled */
#define EEH_STATE_DMA_ENABLED (1 << 6) /* DMA enabled */
+#define EEH_PE_STATE_NORMAL 0 /* Normal state */
+#define EEH_PE_STATE_RESET 1 /* PE reset asserted */
+#define EEH_PE_STATE_STOPPED_IO_DMA 2 /* Frozen PE */
+#define EEH_PE_STATE_STOPPED_DMA 4 /* Stopped DMA, Enabled IO */
+#define EEH_PE_STATE_UNAVAIL 5 /* Unavailable */
#define EEH_RESET_DEACTIVATE 0 /* Deactivate the PE reset */
#define EEH_RESET_HOT 1 /* Hot reset */
#define EEH_RESET_FUNDAMENTAL 3 /* Fundamental reset */
@@ -280,6 +285,13 @@ void eeh_add_device_late(struct pci_dev *);
void eeh_add_device_tree_late(struct pci_bus *);
void eeh_add_sysfs_files(struct pci_bus *);
void eeh_remove_device(struct pci_dev *);
+int eeh_dev_open(struct pci_dev *pdev);
+void eeh_dev_release(struct pci_dev *pdev);
+struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group);
+int eeh_pe_set_option(struct eeh_pe *pe, int option);
+int eeh_pe_get_state(struct eeh_pe *pe);
+int eeh_pe_reset(struct eeh_pe *pe, int option);
+int eeh_pe_configure(struct eeh_pe *pe);
/**
* EEH_POSSIBLE_ERROR() -- test for possible MMIO failure.
diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
index 3bc8b12..fc90df0 100644
--- a/arch/powerpc/kernel/eeh.c
+++ b/arch/powerpc/kernel/eeh.c
@@ -40,6 +40,7 @@
#include <asm/eeh.h>
#include <asm/eeh_event.h>
#include <asm/io.h>
+#include <asm/iommu.h>
#include <asm/machdep.h>
#include <asm/ppc-pci.h>
#include <asm/rtas.h>
@@ -108,6 +109,9 @@ struct eeh_ops *eeh_ops = NULL;
/* Lock to avoid races due to multiple reports of an error */
DEFINE_RAW_SPINLOCK(confirm_error_lock);
+/* Lock to protect passed flags */
+static DEFINE_MUTEX(eeh_dev_mutex);
+
/* Buffer for reporting pci register dumps. Its here in BSS, and
* not dynamically alloced, so that it ends up in RMO where RTAS
* can access it.
@@ -1106,6 +1110,270 @@ void eeh_remove_device(struct pci_dev *dev)
edev->mode &= ~EEH_DEV_SYSFS;
}
+/**
+ * eeh_dev_open - Increase count of pass through devices for PE
+ * @pdev: PCI device
+ *
+ * Increase count of passed through devices for the indicated
+ * PE. In the result, the EEH errors detected on the PE won't be
+ * reported. The PE owner will be responsible for detection
+ * and recovery.
+ */
+int eeh_dev_open(struct pci_dev *pdev)
+{
+ struct eeh_dev *edev;
+
+ mutex_lock(&eeh_dev_mutex);
+
+ /* No PCI device ? */
+ if (!pdev)
+ goto out;
+
+ /* No EEH device or PE ? */
+ edev = pci_dev_to_eeh_dev(pdev);
+ if (!edev || !edev->pe)
+ goto out;
+
+ /* Increase PE's pass through count */
+ atomic_inc(&edev->pe->pass_dev_cnt);
+ mutex_unlock(&eeh_dev_mutex);
+
+ return 0;
+out:
+ mutex_unlock(&eeh_dev_mutex);
+ return -ENODEV;
+}
+EXPORT_SYMBOL_GPL(eeh_dev_open);
+
+/**
+ * eeh_dev_release - Decrease count of pass through devices for PE
+ * @pdev: PCI device
+ *
+ * Decrease count of pass through devices for the indicated PE. If
+ * there is no passed through device in PE, the EEH errors detected
+ * on the PE will be reported and handled as usual.
+ */
+void eeh_dev_release(struct pci_dev *pdev)
+{
+ struct eeh_dev *edev;
+
+ mutex_lock(&eeh_dev_mutex);
+
+ /* No PCI device ? */
+ if (!pdev)
+ goto out;
+
+ /* No EEH device ? */
+ edev = pci_dev_to_eeh_dev(pdev);
+ if (!edev || !edev->pe || !eeh_pe_passed(edev->pe))
+ goto out;
+
+ /* Decrease PE's pass through count */
+ atomic_dec(&edev->pe->pass_dev_cnt);
+ WARN_ON(atomic_read(&edev->pe->pass_dev_cnt) < 0);
+out:
+ mutex_unlock(&eeh_dev_mutex);
+}
+EXPORT_SYMBOL(eeh_dev_release);
+
+/**
+ * eeh_iommu_group_to_pe - Convert IOMMU group to EEH PE
+ * @group: IOMMU group
+ *
+ * The routine is called to convert IOMMU group to EEH PE.
+ */
+struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group)
+{
+ struct iommu_table *tbl;
+ struct pci_dev *pdev = NULL;
+ struct eeh_dev *edev;
+ bool found = false;
+
+ /* No IOMMU group ? */
+ if (!group)
+ return NULL;
+
+ /* No PCI device ? */
+ for_each_pci_dev(pdev) {
+ tbl = get_iommu_table_base(&pdev->dev);
+ if (tbl && tbl->it_group == group) {
+ found = true;
+ break;
+ }
+ }
+ if (!found)
+ return NULL;
+
+ /* No EEH device or PE ? */
+ edev = pci_dev_to_eeh_dev(pdev);
+ if (!edev || !edev->pe)
+ return NULL;
+
+ return edev->pe;
+}
+
+/**
+ * eeh_pe_set_option - Set options for the indicated PE
+ * @pe: EEH PE
+ * @option: requested option
+ *
+ * The routine is called to enable or disable EEH functionality
+ * on the indicated PE, to enable IO or DMA for the frozen PE.
+ */
+int eeh_pe_set_option(struct eeh_pe *pe, int option)
+{
+ int ret = 0;
+
+ /* Invalid PE ? */
+ if (!pe)
+ return -ENODEV;
+
+ /*
+ * EEH functionality could possibly be disabled, just
+ * return error for the case. And the EEH functinality
+ * isn't expected to be disabled on one specific PE.
+ */
+ switch (option) {
+ case EEH_OPT_ENABLE:
+ if (eeh_enabled())
+ break;
+ ret = -EIO;
+ break;
+ case EEH_OPT_DISABLE:
+ break;
+ case EEH_OPT_THAW_MMIO:
+ case EEH_OPT_THAW_DMA:
+ if (!eeh_ops || !eeh_ops->set_option) {
+ ret = -ENOENT;
+ break;
+ }
+
+ ret = eeh_ops->set_option(pe, option);
+ break;
+ default:
+ pr_debug("%s: Option %d out of range (%d, %d)\n",
+ __func__, option, EEH_OPT_DISABLE, EEH_OPT_THAW_DMA);
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(eeh_pe_set_option);
+
+/**
+ * eeh_pe_get_state - Retrieve PE's state
+ * @pe: EEH PE
+ *
+ * Retrieve the PE's state, which includes 3 aspects: enabled
+ * DMA, enabled IO and asserted reset.
+ */
+int eeh_pe_get_state(struct eeh_pe *pe)
+{
+ int result, ret = 0;
+ bool rst_active, dma_en, mmio_en;
+
+ /* Existing PE ? */
+ if (!pe)
+ return -ENODEV;
+
+ if (!eeh_ops || !eeh_ops->get_state)
+ return -ENOENT;
+
+ result = eeh_ops->get_state(pe, NULL);
+ rst_active = !!(result & EEH_STATE_RESET_ACTIVE);
+ dma_en = !!(result & EEH_STATE_DMA_ENABLED);
+ mmio_en = !!(result & EEH_STATE_MMIO_ENABLED);
+
+ if (rst_active)
+ ret = EEH_PE_STATE_RESET;
+ else if (dma_en && mmio_en)
+ ret = EEH_PE_STATE_NORMAL;
+ else if (!dma_en && !mmio_en)
+ ret = EEH_PE_STATE_STOPPED_IO_DMA;
+ else if (!dma_en && mmio_en)
+ ret = EEH_PE_STATE_STOPPED_DMA;
+ else
+ ret = EEH_PE_STATE_UNAVAIL;
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(eeh_pe_get_state);
+
+/**
+ * eeh_pe_reset - Issue PE reset according to specified type
+ * @pe: EEH PE
+ * @option: reset type
+ *
+ * The routine is called to reset the specified PE with the
+ * indicated type, either fundamental reset or hot reset.
+ * PE reset is the most important part for error recovery.
+ */
+int eeh_pe_reset(struct eeh_pe *pe, int option)
+{
+ int ret = 0;
+
+ /* Invalid PE ? */
+ if (!pe)
+ return -ENODEV;
+
+ if (!eeh_ops || !eeh_ops->set_option || !eeh_ops->reset)
+ return -ENOENT;
+
+ switch (option) {
+ case EEH_RESET_DEACTIVATE:
+ ret = eeh_ops->reset(pe, option);
+ if (ret)
+ break;
+
+ /*
+ * The PE is still in frozen state and we need to clear
+ * that. It's good to clear frozen state after deassert
+ * to avoid messy IO access during reset, which might
+ * cause recursive frozen PE.
+ */
+ ret = eeh_ops->set_option(pe, EEH_OPT_THAW_MMIO);
+ if (!ret)
+ ret = eeh_ops->set_option(pe, EEH_OPT_THAW_DMA);
+ if (!ret)
+ eeh_pe_state_clear(pe, EEH_PE_ISOLATED);
+ break;
+ case EEH_RESET_HOT:
+ case EEH_RESET_FUNDAMENTAL:
+ ret = eeh_ops->reset(pe, option);
+ break;
+ default:
+ pr_debug("%s: Unsupported option %d\n",
+ __func__, option);
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(eeh_pe_reset);
+
+/**
+ * eeh_pe_configure - Configure PCI bridges after PE reset
+ * @pe: EEH PE
+ *
+ * The routine is called to restore the PCI config space for
+ * those PCI devices, especially PCI bridges affected by PE
+ * reset issued previously.
+ */
+int eeh_pe_configure(struct eeh_pe *pe)
+{
+ int ret = 0;
+
+ /* Invalid PE ? */
+ if (!pe)
+ return -ENODEV;
+
+ /* Restore config space for the affected devices */
+ eeh_pe_restore_bars(pe);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(eeh_pe_configure);
+
static int proc_eeh_show(struct seq_file *m, void *v)
{
if (!eeh_enabled()) {
--
1.8.3.2
^ permalink raw reply related
* [RFC PATCH V3 00/17] Enable SRIOV on POWER8
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
This patch set enable the SRIOV on POWER8. This is not the final version, some
patches rely on un-merged patches.
The gerneral idea is put each VF in their own PE and allocated necessary
resources, like DMA/IOMMU_TABLE.
One thing special for VF PE is we use M64BT to cover the IOV BAR. This means
we need to do some hack on pci devices's resources.
1. Expand the IOV BAR properly.
2. Shift the IOV BAR properly.
3. IOV BAR alignment is the total size instead of an individual size.
4. Take the IOV BAR alignment into consideration in the sizing and assigning.
Test Environment:
The SRIOV device tested is Emulex Lancer and Mellanox ConnectX-3.
Examples on pass through a VF to guest through vfio:
1. install necessary modules
modprobe vfio
modprobe vfio-pci
2. retrieve the iommu_group the device belongs to
readlink /sys/bus/pci/devices/0000:06:0d.0/iommu_group
../../../../kernel/iommu_groups/26
This means it belongs to group 26
3. see how many devices under this iommu_group
ls ls /sys/kernel/iommu_groups/26/devices/
4. unbind the original driver and bind to vfio-pci driver
echo 0000:06:0d.0 > /sys/bus/pci/devices/0000:06:0d.0/driver/unbind
echo 1102 0002 > /sys/bus/pci/drivers/vfio-pci/new_id
Note: this should be done for each device in the same iommu_group
5. Start qemu and pass device through vfio
/home/ywywyang/git/qemu-impreza/ppc64-softmmu/qemu-system-ppc64 \
-M pseries -m 2048 -enable-kvm -nographic \
-drive file=/home/ywywyang/kvm/fc19.img \
-monitor telnet:localhost:5435,server,nowait -boot cd \
-device "spapr-pci-vfio-host-bridge,id=CXGB3,iommu=26,index=6"
Verify this is the exact VF response:
1. ping from a machine in the same subnet(the broadcast domain)
2. run arp -n on this machine
9.115.251.20 ether 00:00:c9:df:ed:bf C eth0
3. ifconfig in the guest
# ifconfig eth1
eth1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 9.115.251.20 netmask 255.255.255.0 broadcast 9.115.251.255
inet6 fe80::200:c9ff:fedf:edbf prefixlen 64 scopeid 0x20<link>
ether 00:00:c9:df:ed:bf txqueuelen 1000 (Ethernet)
RX packets 175 bytes 13278 (12.9 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 58 bytes 9276 (9.0 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
4. They have the same MAC address
Note: make sure you shutdown other network interfaces in guest.
---
v2 -> v3:
1. change the return type of virtfn_bus/virtfn_devfn to int
change the name of these two functions to pci_iov_virtfn_bus/pci_iov_virtfn_devfn
2. reduce the second parameter or pcibios_sriov_disable()
3. use data instead of pe in "ppc/pnv: allocate pe->iommu_table dynamically"
4. rename __pci_sriov_resource_size to pcibios_sriov_resource_size
5. rename __pci_sriov_resource_alignment to pcibios_sriov_resource_alignment
v1 -> v2:
1. change the return value of virtfn_bus/virtfn_devfn to 0
2. move some TCE related marco definition to
arch/powerpc/platforms/powernv/pci.h
3. fix the __pci_sriov_resource_alignment on powernv platform
During the sizing stage, the IOV BAR is truncated to 0, which will
effect the order of allocation. Fix this, so that make sure BAR will be
allocated ordered by their alignment.
v0 -> v1:
1. Improve the change log for
"PCI: Add weak __pci_sriov_resource_size() interface"
"PCI: Add weak __pci_sriov_resource_alignment() interface"
"PCI: take additional IOV BAR alignment in sizing and assigning"
2. Wrap VF PE code in CONFIG_PCI_IOV
3. Did regression test on P7.
Wei Yang (17):
pci/iov: Export interface for retrieve VF's BDF
pci/of: Match PCI VFs to dev-tree nodes dynamically
ppc/pci: don't unset pci resources for VFs
PCI: SRIOV: add VF enable/disable hook
ppc/pnv: user macro to define the TCE size
ppc/pnv: allocate pe->iommu_table dynamically
ppc/pnv: Add function to deconfig a PE
PCI: Add weak pcibios_sriov_resource_size() interface
PCI: Add weak pcibios_sriov_resource_alignment() interface
PCI: take additional IOV BAR alignment in sizing and assigning
ppc/pnv: Expand VF resources according to the number of total_pe
powerpc/powernv: implement pcibios_sriov_resource_alignment on
powernv
powerpc/powernv: shift VF resource with an offset
ppc/pci: create/release dev-tree node for VFs
powerpc/powernv: allocate VF PE
ppc/pci: Expanding IOV BAR, with m64_per_iov supported
ppc/pnv: Group VF PE when IOV BAR is big on PHB3
arch/powerpc/include/asm/iommu.h | 3 +
arch/powerpc/include/asm/machdep.h | 7 +
arch/powerpc/include/asm/pci-bridge.h | 7 +
arch/powerpc/include/asm/tce.h | 3 +-
arch/powerpc/kernel/pci-common.c | 29 +
arch/powerpc/platforms/powernv/Kconfig | 1 +
arch/powerpc/platforms/powernv/pci-ioda.c | 824 +++++++++++++++++++++++++++--
arch/powerpc/platforms/powernv/pci.c | 22 +-
arch/powerpc/platforms/powernv/pci.h | 17 +-
drivers/pci/iov.c | 84 ++-
drivers/pci/pci.h | 21 -
drivers/pci/setup-bus.c | 66 ++-
include/linux/pci.h | 46 ++
13 files changed, 1041 insertions(+), 89 deletions(-)
--
1.7.9.5
^ permalink raw reply
* [RFC PATCH V3 01/17] pci/iov: Export interface for retrieve VF's BDF
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
When implementing the SR-IOV on PowerNV platform, some resource reservation is
needed for VFs which don't exist at the bootup stage. To do the match between
resources and VFs, the code need to get the VF's BDF in advance.
In this patch, it exports the interface to retrieve VF's BDF:
* Make the virtfn_bus as an interface
* Make the virtfn_devfn as an interface
* rename them with more specific name
* code cleanup in pci_sriov_resource_alignment()
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
drivers/pci/iov.c | 26 +++++++-------------------
drivers/pci/pci.h | 21 ---------------------
include/linux/pci.h | 43 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 50 insertions(+), 40 deletions(-)
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 9dce7c5..589ef7d 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -19,18 +19,6 @@
#define VIRTFN_ID_LEN 16
-static inline u8 virtfn_bus(struct pci_dev *dev, int id)
-{
- return dev->bus->number + ((dev->devfn + dev->sriov->offset +
- dev->sriov->stride * id) >> 8);
-}
-
-static inline u8 virtfn_devfn(struct pci_dev *dev, int id)
-{
- return (dev->devfn + dev->sriov->offset +
- dev->sriov->stride * id) & 0xff;
-}
-
static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr)
{
struct pci_bus *child;
@@ -69,7 +57,7 @@ static int virtfn_add(struct pci_dev *dev, int id, int reset)
struct pci_bus *bus;
mutex_lock(&iov->dev->sriov->lock);
- bus = virtfn_add_bus(dev->bus, virtfn_bus(dev, id));
+ bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id));
if (!bus)
goto failed;
@@ -77,7 +65,7 @@ static int virtfn_add(struct pci_dev *dev, int id, int reset)
if (!virtfn)
goto failed0;
- virtfn->devfn = virtfn_devfn(dev, id);
+ virtfn->devfn = pci_iov_virtfn_devfn(dev, id);
virtfn->vendor = dev->vendor;
pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
pci_setup_device(virtfn);
@@ -140,8 +128,8 @@ static void virtfn_remove(struct pci_dev *dev, int id, int reset)
struct pci_sriov *iov = dev->sriov;
virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus),
- virtfn_bus(dev, id),
- virtfn_devfn(dev, id));
+ pci_iov_virtfn_bus(dev, id),
+ pci_iov_virtfn_devfn(dev, id));
if (!virtfn)
return;
@@ -307,7 +295,7 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
iov->offset = offset;
iov->stride = stride;
- if (virtfn_bus(dev, nr_virtfn - 1) > dev->bus->busn_res.end) {
+ if (pci_iov_virtfn_bus(dev, nr_virtfn - 1) > dev->bus->busn_res.end) {
dev_err(&dev->dev, "SR-IOV: bus number out of range\n");
return -ENOMEM;
}
@@ -616,7 +604,7 @@ resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)
if (!reg)
return 0;
- __pci_read_base(dev, type, &tmp, reg);
+ __pci_read_base(dev, type, &tmp, reg);
return resource_alignment(&tmp);
}
@@ -646,7 +634,7 @@ int pci_iov_bus_range(struct pci_bus *bus)
list_for_each_entry(dev, &bus->devices, bus_list) {
if (!dev->is_physfn)
continue;
- busnr = virtfn_bus(dev, dev->sriov->total_VFs - 1);
+ busnr = pci_iov_virtfn_bus(dev, dev->sriov->total_VFs - 1);
if (busnr > max)
max = busnr;
}
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 4df38df..51f1f7c 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -223,27 +223,6 @@ static inline int pci_ari_enabled(struct pci_bus *bus)
void pci_reassigndev_resource_alignment(struct pci_dev *dev);
void pci_disable_bridge_window(struct pci_dev *dev);
-/* Single Root I/O Virtualization */
-struct pci_sriov {
- int pos; /* capability position */
- int nres; /* number of resources */
- u32 cap; /* SR-IOV Capabilities */
- u16 ctrl; /* SR-IOV Control */
- u16 total_VFs; /* total VFs associated with the PF */
- u16 initial_VFs; /* initial VFs associated with the PF */
- u16 num_VFs; /* number of VFs available */
- u16 offset; /* first VF Routing ID offset */
- u16 stride; /* following VF stride */
- u32 pgsz; /* page size for BAR alignment */
- u8 link; /* Function Dependency Link */
- u16 driver_max_VFs; /* max num VFs driver supports */
- struct pci_dev *dev; /* lowest numbered PF */
- struct pci_dev *self; /* this PF */
- struct mutex lock; /* lock for VF bus */
- struct work_struct mtask; /* VF Migration task */
- u8 __iomem *mstate; /* VF Migration State Array */
-};
-
#ifdef CONFIG_PCI_ATS
void pci_restore_ats_state(struct pci_dev *dev);
#else
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 33aa2ca..ddb1ca0 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -240,6 +240,27 @@ struct pci_vpd;
struct pci_sriov;
struct pci_ats;
+/* Single Root I/O Virtualization */
+struct pci_sriov {
+ int pos; /* capability position */
+ int nres; /* number of resources */
+ u32 cap; /* SR-IOV Capabilities */
+ u16 ctrl; /* SR-IOV Control */
+ u16 total_VFs; /* total VFs associated with the PF */
+ u16 initial_VFs; /* initial VFs associated with the PF */
+ u16 num_VFs; /* number of VFs available */
+ u16 offset; /* first VF Routing ID offset */
+ u16 stride; /* following VF stride */
+ u32 pgsz; /* page size for BAR alignment */
+ u8 link; /* Function Dependency Link */
+ u16 driver_max_VFs; /* max num VFs driver supports */
+ struct pci_dev *dev; /* lowest numbered PF */
+ struct pci_dev *self; /* this PF */
+ struct mutex lock; /* lock for VF bus */
+ struct work_struct mtask; /* VF Migration task */
+ u8 __iomem *mstate; /* VF Migration State Array */
+};
+
/*
* The pci_dev structure is used to describe PCI devices.
*/
@@ -1595,6 +1616,20 @@ int pci_ext_cfg_avail(void);
void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar);
#ifdef CONFIG_PCI_IOV
+static inline int pci_iov_virtfn_bus(struct pci_dev *dev, int id)
+{
+ if (!dev->is_physfn)
+ return -1;
+ return dev->bus->number + ((dev->devfn + dev->sriov->offset +
+ dev->sriov->stride * id) >> 8);
+}
+static inline int pci_iov_virtfn_devfn(struct pci_dev *dev, int id)
+{
+ if (!dev->is_physfn)
+ return -1;
+ return (dev->devfn + dev->sriov->offset +
+ dev->sriov->stride * id) & 0xff;
+}
int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn);
void pci_disable_sriov(struct pci_dev *dev);
irqreturn_t pci_sriov_migration(struct pci_dev *dev);
@@ -1603,6 +1638,14 @@ int pci_vfs_assigned(struct pci_dev *dev);
int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs);
int pci_sriov_get_totalvfs(struct pci_dev *dev);
#else
+static inline int pci_iov_virtfn_bus(struct pci_dev *dev, int id)
+{
+ return -1;
+}
+static inline int pci_iov_virtfn_devfn(struct pci_dev *dev, int id)
+{
+ return -1;
+}
static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
{ return -ENODEV; }
static inline void pci_disable_sriov(struct pci_dev *dev) { }
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 02/17] pci/of: Match PCI VFs to dev-tree nodes dynamically
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
As introduced by commit 98d9f30c82 ("pci/of: Match PCI devices to dev-tree nodes
dynamically"), we need to match PCI devices to their corresponding dev-tree
nodes. While for VFs, this step was missed.
This patch matches VFs' PCI devices to dev-tree nodes dynamically.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
drivers/pci/iov.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 589ef7d..1d21f43 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -67,6 +67,7 @@ static int virtfn_add(struct pci_dev *dev, int id, int reset)
virtfn->devfn = pci_iov_virtfn_devfn(dev, id);
virtfn->vendor = dev->vendor;
+ pci_set_of_node(virtfn);
pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_DID, &virtfn->device);
pci_setup_device(virtfn);
virtfn->dev.parent = dev->dev.parent;
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 04/17] PCI: SRIOV: add VF enable/disable hook
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
VFs are dynamically created/released when driver enable them. On some
platforms, like PowerNV, special resources are necessary to enable VFs.
This patch adds two hooks for platform initialization before creating the VFs.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
drivers/pci/iov.c | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 1d21f43..cc87773 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -250,6 +250,11 @@ static void sriov_disable_migration(struct pci_dev *dev)
iounmap(iov->mstate);
}
+int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 vf_num)
+{
+ return 0;
+}
+
static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
{
int rc;
@@ -260,6 +265,7 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
struct pci_dev *pdev;
struct pci_sriov *iov = dev->sriov;
int bars = 0;
+ int retval;
if (!nr_virtfn)
return 0;
@@ -334,6 +340,12 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
if (nr_virtfn < initial)
initial = nr_virtfn;
+ if ((retval = pcibios_sriov_enable(dev, initial))) {
+ dev_err(&dev->dev, "Failure %d from pcibios_sriov_setup()\n",
+ retval);
+ return retval;
+ }
+
for (i = 0; i < initial; i++) {
rc = virtfn_add(dev, i, 0);
if (rc)
@@ -368,6 +380,11 @@ failed:
return rc;
}
+int __weak pcibios_sriov_disable(struct pci_dev *pdev)
+{
+ return 0;
+}
+
static void sriov_disable(struct pci_dev *dev)
{
int i;
@@ -382,6 +399,8 @@ static void sriov_disable(struct pci_dev *dev)
for (i = 0; i < iov->num_VFs; i++)
virtfn_remove(dev, i, 0);
+ pcibios_sriov_disable(dev);
+
iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
pci_cfg_access_lock(dev);
pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 03/17] ppc/pci: don't unset pci resources for VFs
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
When the PCI_REASSIGN_ALL_RSRC is set, each resource for a pci_dev will be
unset. which means the pci core will reassign those resources.
While this behavior will clean up the resources information for VFs, whose
value is calculated in virtfn_add.
This patch adds a condition. If the pci_dev is a VF, skip the resource
unset process.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/kernel/pci-common.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c
index d9476c1..c449a26 100644
--- a/arch/powerpc/kernel/pci-common.c
+++ b/arch/powerpc/kernel/pci-common.c
@@ -824,6 +824,12 @@ static void pcibios_fixup_resources(struct pci_dev *dev)
pci_name(dev));
return;
}
+
+#ifdef CONFIG_PCI_IOV
+ if (dev->is_virtfn)
+ return;
+#endif
+
for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
struct resource *res = dev->resource + i;
struct pci_bus_region reg;
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 05/17] ppc/pnv: user macro to define the TCE size
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
During the initialization of the TVT/TCE, it uses digits to specify the TCE IO
Page Size, TCE Table Size, TCE Entry Size, etc.
This patch replaces those digits with macros, which will be more meaningful and
easy to read.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/tce.h | 3 ++-
arch/powerpc/platforms/powernv/pci-ioda.c | 25 +++++++++++--------------
arch/powerpc/platforms/powernv/pci.c | 2 +-
arch/powerpc/platforms/powernv/pci.h | 5 +++++
4 files changed, 19 insertions(+), 16 deletions(-)
diff --git a/arch/powerpc/include/asm/tce.h b/arch/powerpc/include/asm/tce.h
index 743f36b..28a1d06 100644
--- a/arch/powerpc/include/asm/tce.h
+++ b/arch/powerpc/include/asm/tce.h
@@ -40,7 +40,8 @@
#define TCE_SHIFT 12
#define TCE_PAGE_SIZE (1 << TCE_SHIFT)
-#define TCE_ENTRY_SIZE 8 /* each TCE is 64 bits */
+#define TCE_ENTRY_SHIFT 3
+#define TCE_ENTRY_SIZE (1 << TCE_ENTRY_SHIFT) /* each TCE is 64 bits */
#define TCE_RPN_MASK 0xfffffffffful /* 40-bit RPN (4K pages) */
#define TCE_RPN_SHIFT 12
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 8ae09cf..9715351 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -820,9 +820,6 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
int64_t rc;
void *addr;
- /* 256M DMA window, 4K TCE pages, 8 bytes TCE */
-#define TCE32_TABLE_SIZE ((0x10000000 / 0x1000) * 8)
-
/* XXX FIXME: Handle 64-bit only DMA devices */
/* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */
/* XXX FIXME: Allocate multi-level tables on PHB3 */
@@ -834,7 +831,7 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
/* Grab a 32-bit TCE table */
pe->tce32_seg = base;
pe_info(pe, " Setting up 32-bit TCE table at %08x..%08x\n",
- (base << 28), ((base + segs) << 28) - 1);
+ (base << PNV_TCE32_SEG_SHIFT), ((base + segs) << PNV_TCE32_SEG_SHIFT) - 1);
/* XXX Currently, we allocate one big contiguous table for the
* TCEs. We only really need one chunk per 256M of TCE space
@@ -842,21 +839,21 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
* requires some added smarts with our get/put_tce implementation
*/
tce_mem = alloc_pages_node(phb->hose->node, GFP_KERNEL,
- get_order(TCE32_TABLE_SIZE * segs));
+ get_order(PNV_TCE32_TAB_SIZE * segs));
if (!tce_mem) {
pe_err(pe, " Failed to allocate a 32-bit TCE memory\n");
goto fail;
}
addr = page_address(tce_mem);
- memset(addr, 0, TCE32_TABLE_SIZE * segs);
+ memset(addr, 0, PNV_TCE32_TAB_SIZE * segs);
/* Configure HW */
for (i = 0; i < segs; i++) {
rc = opal_pci_map_pe_dma_window(phb->opal_id,
pe->pe_number,
base + i, 1,
- __pa(addr) + TCE32_TABLE_SIZE * i,
- TCE32_TABLE_SIZE, 0x1000);
+ __pa(addr) + PNV_TCE32_TAB_SIZE * i,
+ PNV_TCE32_TAB_SIZE, TCE_PAGE_SIZE);
if (rc) {
pe_err(pe, " Failed to configure 32-bit TCE table,"
" err %ld\n", rc);
@@ -866,8 +863,8 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
/* Setup linux iommu table */
tbl = &pe->tce32_table;
- pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,
- base << 28);
+ pnv_pci_setup_iommu_table(tbl, addr, PNV_TCE32_TAB_SIZE * segs,
+ base << PNV_TCE32_SEG_SHIFT);
/* OPAL variant of P7IOC SW invalidated TCEs */
swinvp = of_get_property(phb->hose->dn, "ibm,opal-tce-kill", NULL);
@@ -898,7 +895,7 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
if (pe->tce32_seg >= 0)
pe->tce32_seg = -1;
if (tce_mem)
- __free_pages(tce_mem, get_order(TCE32_TABLE_SIZE * segs));
+ __free_pages(tce_mem, get_order(PNV_TCE32_TAB_SIZE * segs));
}
static void pnv_pci_ioda2_set_bypass(struct iommu_table *tbl, bool enable)
@@ -968,7 +965,7 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
/* The PE will reserve all possible 32-bits space */
pe->tce32_seg = 0;
end = (1 << ilog2(phb->ioda.m32_pci_base));
- tce_table_size = (end / 0x1000) * 8;
+ tce_table_size = (end / TCE_PAGE_SIZE) * TCE_ENTRY_SIZE;
pe_info(pe, "Setting up 32-bit TCE table at 0..%08x\n",
end);
@@ -988,7 +985,7 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
*/
rc = opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number,
pe->pe_number << 1, 1, __pa(addr),
- tce_table_size, 0x1000);
+ tce_table_size, TCE_PAGE_SIZE);
if (rc) {
pe_err(pe, "Failed to configure 32-bit TCE table,"
" err %ld\n", rc);
@@ -1573,7 +1570,7 @@ void __init pnv_pci_init_ioda_phb(struct device_node *np,
INIT_LIST_HEAD(&phb->ioda.pe_list);
/* Calculate how many 32-bit TCE segments we have */
- phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28;
+ phb->ioda.tce32_count = phb->ioda.m32_pci_base >> PNV_TCE32_SEG_SHIFT;
#if 0 /* We should really do that ... */
rc = opal_pci_set_phb_mem_window(opal->phb_id,
diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c
index 8518817..687a068 100644
--- a/arch/powerpc/platforms/powernv/pci.c
+++ b/arch/powerpc/platforms/powernv/pci.c
@@ -597,7 +597,7 @@ void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K;
tbl->it_offset = dma_offset >> tbl->it_page_shift;
tbl->it_index = 0;
- tbl->it_size = tce_size >> 3;
+ tbl->it_size = tce_size >> TCE_ENTRY_SHIFT;
tbl->it_busno = 0;
tbl->it_type = TCE_PCI;
}
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index 3e5f5a1..90f6da4 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -227,4 +227,9 @@ extern void pnv_pci_init_ioda2_phb(struct device_node *np);
extern void pnv_pci_ioda_tce_invalidate(struct iommu_table *tbl,
__be64 *startp, __be64 *endp, bool rm);
+#define PNV_TCE32_SEG_SHIFT 28
+#define PNV_TCE32_SEG_SIZE (1UL << PNV_TCE32_SEG_SHIFT)
+/* 256M DMA window, 4K TCE pages, 8 bytes TCE */
+#define PNV_TCE32_TAB_SIZE ((PNV_TCE32_SEG_SIZE / TCE_PAGE_SIZE) * TCE_ENTRY_SIZE)
+
#endif /* __POWERNV_PCI_H */
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 06/17] ppc/pnv: allocate pe->iommu_table dynamically
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
Current iommu_table of a PE is a static field. This will have a problem when
iommu_free_table is called.
This patch allocate iommu_table dynamically.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/iommu.h | 3 +++
arch/powerpc/platforms/powernv/pci-ioda.c | 24 +++++++++++++-----------
arch/powerpc/platforms/powernv/pci.h | 2 +-
3 files changed, 17 insertions(+), 12 deletions(-)
diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h
index 42632c7..0fedacb 100644
--- a/arch/powerpc/include/asm/iommu.h
+++ b/arch/powerpc/include/asm/iommu.h
@@ -78,6 +78,9 @@ struct iommu_table {
struct iommu_group *it_group;
#endif
void (*set_bypass)(struct iommu_table *tbl, bool enable);
+#ifdef CONFIG_PPC_POWERNV
+ void *data;
+#endif
};
/* Pure 2^n version of get_order */
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 9715351..8ca3926 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -608,6 +608,10 @@ static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all)
return;
}
+ pe->tce32_table = kzalloc_node(sizeof(struct iommu_table),
+ GFP_KERNEL, hose->node);
+ pe->tce32_table->data = pe;
+
/* Associate it with all child devices */
pnv_ioda_setup_same_PE(bus, pe);
@@ -675,7 +679,7 @@ static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev
pe = &phb->ioda.pe_array[pdn->pe_number];
WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops);
- set_iommu_table_base_and_group(&pdev->dev, &pe->tce32_table);
+ set_iommu_table_base_and_group(&pdev->dev, pe->tce32_table);
}
static int pnv_pci_ioda_dma_set_mask(struct pnv_phb *phb,
@@ -702,7 +706,7 @@ static int pnv_pci_ioda_dma_set_mask(struct pnv_phb *phb,
} else {
dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n");
set_dma_ops(&pdev->dev, &dma_iommu_ops);
- set_iommu_table_base(&pdev->dev, &pe->tce32_table);
+ set_iommu_table_base(&pdev->dev, pe->tce32_table);
}
return 0;
}
@@ -712,7 +716,7 @@ static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe, struct pci_bus *bus)
struct pci_dev *dev;
list_for_each_entry(dev, &bus->devices, bus_list) {
- set_iommu_table_base_and_group(&dev->dev, &pe->tce32_table);
+ set_iommu_table_base_and_group(&dev->dev, pe->tce32_table);
if (dev->subordinate)
pnv_ioda_setup_bus_dma(pe, dev->subordinate);
}
@@ -798,8 +802,7 @@ static void pnv_pci_ioda2_tce_invalidate(struct pnv_ioda_pe *pe,
void pnv_pci_ioda_tce_invalidate(struct iommu_table *tbl,
__be64 *startp, __be64 *endp, bool rm)
{
- struct pnv_ioda_pe *pe = container_of(tbl, struct pnv_ioda_pe,
- tce32_table);
+ struct pnv_ioda_pe *pe = tbl->data;
struct pnv_phb *phb = pe->phb;
if (phb->type == PNV_PHB_IODA1)
@@ -862,7 +865,7 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
}
/* Setup linux iommu table */
- tbl = &pe->tce32_table;
+ tbl = pe->tce32_table;
pnv_pci_setup_iommu_table(tbl, addr, PNV_TCE32_TAB_SIZE * segs,
base << PNV_TCE32_SEG_SHIFT);
@@ -900,8 +903,7 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
static void pnv_pci_ioda2_set_bypass(struct iommu_table *tbl, bool enable)
{
- struct pnv_ioda_pe *pe = container_of(tbl, struct pnv_ioda_pe,
- tce32_table);
+ struct pnv_ioda_pe *pe = tbl->data;
uint16_t window_id = (pe->pe_number << 1 ) + 1;
int64_t rc;
@@ -942,10 +944,10 @@ static void pnv_pci_ioda2_setup_bypass_pe(struct pnv_phb *phb,
pe->tce_bypass_base = 1ull << 59;
/* Install set_bypass callback for VFIO */
- pe->tce32_table.set_bypass = pnv_pci_ioda2_set_bypass;
+ pe->tce32_table->set_bypass = pnv_pci_ioda2_set_bypass;
/* Enable bypass by default */
- pnv_pci_ioda2_set_bypass(&pe->tce32_table, true);
+ pnv_pci_ioda2_set_bypass(pe->tce32_table, true);
}
static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
@@ -993,7 +995,7 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
}
/* Setup linux iommu table */
- tbl = &pe->tce32_table;
+ tbl = pe->tce32_table;
pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, 0);
/* OPAL variant of PHB3 invalidated TCEs */
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index 90f6da4..9fbf7c0 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -60,7 +60,7 @@ struct pnv_ioda_pe {
/* "Base" iommu table, ie, 4K TCEs, 32-bit DMA */
int tce32_seg;
int tce32_segcount;
- struct iommu_table tce32_table;
+ struct iommu_table *tce32_table;
phys_addr_t tce_inval_reg_phys;
/* 64-bit TCE bypass region */
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 07/17] ppc/pnv: Add function to deconfig a PE
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
On PowerNV platform, it will support dynamic PE allocation and deallocation.
This patch adds a function to release those resources related to a PE.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/platforms/powernv/pci-ioda.c | 77 +++++++++++++++++++++++++++++
1 file changed, 77 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 8ca3926..87cb3089 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -330,6 +330,83 @@ static struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev)
}
#endif /* CONFIG_PCI_MSI */
+static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
+{
+ struct pci_dev *parent;
+ uint8_t bcomp, dcomp, fcomp;
+ int64_t rc;
+ long rid_end, rid;
+ if (pe->pbus) {
+ int count;
+
+ dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER;
+ fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER;
+ parent = pe->pbus->self;
+ if (pe->flags & PNV_IODA_PE_BUS_ALL)
+ count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1;
+ else
+ count = 1;
+
+ switch(count) {
+ case 1: bcomp = OpalPciBusAll; break;
+ case 2: bcomp = OpalPciBus7Bits; break;
+ case 4: bcomp = OpalPciBus6Bits; break;
+ case 8: bcomp = OpalPciBus5Bits; break;
+ case 16: bcomp = OpalPciBus4Bits; break;
+ case 32: bcomp = OpalPciBus3Bits; break;
+ default:
+ pr_err("%s: Number of subordinate busses %d"
+ " unsupported\n",
+ pci_name(pe->pbus->self), count);
+ /* Do an exact match only */
+ bcomp = OpalPciBusAll;
+ }
+ rid_end = pe->rid + (count << 8);
+ }else {
+ parent = pe->pdev->bus->self;
+ bcomp = OpalPciBusAll;
+ dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
+ fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
+ rid_end = pe->rid + 1;
+ }
+
+ /* Disable MVT on IODA1 */
+ if (phb->type == PNV_PHB_IODA1) {
+ rc = opal_pci_set_mve_enable(phb->opal_id,
+ pe->mve_number, OPAL_DISABLE_MVE);
+ if (rc) {
+ pe_err(pe, "OPAL error %ld enabling MVE %d\n",
+ rc, pe->mve_number);
+ pe->mve_number = -1;
+ }
+ }
+ /* Clear the reverse map */
+ for (rid = pe->rid; rid < rid_end; rid++)
+ phb->ioda.pe_rmap[rid] = 0;
+
+ /* Release from all parents PELT-V */
+ while (parent) {
+ struct pci_dn *pdn = pci_get_pdn(parent);
+ if (pdn && pdn->pe_number != IODA_INVALID_PE) {
+ rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number,
+ pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN);
+ /* XXX What to do in case of error ? */
+ }
+ parent = parent->bus->self;
+ }
+
+ /* Dissociate PE in PELT */
+ rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid,
+ bcomp, dcomp, fcomp, OPAL_UNMAP_PE);
+ if (rc)
+ pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc);
+
+ pe->pbus = NULL;
+ pe->pdev = NULL;
+
+ return 0;
+}
+
static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
{
struct pci_dev *parent;
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 08/17] PCI: Add weak pcibios_sriov_resource_size() interface
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
When retrieving sriov resource size in pci_sriov_resource_size(), it will
divide the total IOV resource size with the totalVF number. This is true for
most cases, while may not be correct on some specific platform.
For example on powernv platform, in order to fix the IOV BAR into a hardware
alignment, the IOV resource size would be expended. This means the original
method couldn't work.
This patch introduces a weak pcibios_sriov_resource_size() interface, which
gives platform a chance to implement specific method to calculate the sriov
resource size.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
drivers/pci/iov.c | 27 +++++++++++++++++++++++++--
include/linux/pci.h | 3 +++
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index cc87773..9fd4648 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -45,6 +45,30 @@ static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus)
pci_remove_bus(virtbus);
}
+resource_size_t __weak pcibios_sriov_resource_size(struct pci_dev *dev, int resno)
+{
+ return 0;
+}
+
+resource_size_t pci_sriov_resource_size(struct pci_dev *dev, int resno)
+{
+ u64 size;
+ struct pci_sriov *iov;
+
+ if (!dev->is_physfn)
+ return 0;
+
+ size = pcibios_sriov_resource_size(dev, resno);
+ if (size != 0)
+ return size;
+
+ iov = dev->sriov;
+ size = resource_size(dev->resource + resno);
+ do_div(size, iov->total_VFs);
+
+ return size;
+}
+
static int virtfn_add(struct pci_dev *dev, int id, int reset)
{
int i;
@@ -81,8 +105,7 @@ static int virtfn_add(struct pci_dev *dev, int id, int reset)
continue;
virtfn->resource[i].name = pci_name(virtfn);
virtfn->resource[i].flags = res->flags;
- size = resource_size(res);
- do_div(size, iov->total_VFs);
+ size = pci_sriov_resource_size(dev, i + PCI_IOV_RESOURCES);
virtfn->resource[i].start = res->start + size * id;
virtfn->resource[i].end = virtfn->resource[i].start + size - 1;
rc = request_resource(res, &virtfn->resource[i]);
diff --git a/include/linux/pci.h b/include/linux/pci.h
index ddb1ca0..315c150 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -1637,6 +1637,7 @@ int pci_num_vf(struct pci_dev *dev);
int pci_vfs_assigned(struct pci_dev *dev);
int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs);
int pci_sriov_get_totalvfs(struct pci_dev *dev);
+resource_size_t pci_sriov_resource_size(struct pci_dev *dev, int resno);
#else
static inline int pci_iov_virtfn_bus(struct pci_dev *dev, int id)
{
@@ -1658,6 +1659,8 @@ static inline int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs)
{ return 0; }
static inline int pci_sriov_get_totalvfs(struct pci_dev *dev)
{ return 0; }
+static inline resource_size_t pci_sriov_resource_size(struct pci_dev *dev, int resno)
+{ return -1; }
#endif
#if defined(CONFIG_HOTPLUG_PCI) || defined(CONFIG_HOTPLUG_PCI_MODULE)
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 09/17] PCI: Add weak pcibios_sriov_resource_alignment() interface
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
The sriov resource alignment is designed to be the individual size of a sriov
resource. This works fine for many platforms, but on powernv platform it needs
some change.
The original alignment works, since at sizing and assigning stage the
requirement is from an individual VF's resource size instead of the big IOV
BAR. This is the reason for the original code to just retrieve the individual
sriov size as the alignment.
On powernv platform, it is required to align the whole IOV BAR to a hardware
aperture. Based on this fact, the alignment of sriov resource should be the
total size of the IOV BAR.
This patch introduces a weak pcibios_sriov_resource_alignment() interface, which
gives platform a chance to implement specific method to calculate the sriov
resource alignment.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
drivers/pci/iov.c | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 9fd4648..dd7fc42 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -628,6 +628,12 @@ int pci_iov_resource_bar(struct pci_dev *dev, int resno,
4 * (resno - PCI_IOV_RESOURCES);
}
+resource_size_t __weak pcibios_sriov_resource_alignment(struct pci_dev *dev,
+ int resno, resource_size_t align)
+{
+ return align;
+}
+
/**
* pci_sriov_resource_alignment - get resource alignment for VF BAR
* @dev: the PCI device
@@ -642,13 +648,16 @@ resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)
{
struct resource tmp;
enum pci_bar_type type;
+ resource_size_t align;
int reg = pci_iov_resource_bar(dev, resno, &type);
if (!reg)
return 0;
__pci_read_base(dev, type, &tmp, reg);
- return resource_alignment(&tmp);
+ align = resource_alignment(&tmp);
+
+ return pcibios_sriov_resource_alignment(dev, resno, align);
}
/**
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 10/17] PCI: take additional IOV BAR alignment in sizing and assigning
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
At resource sizing/assigning stage, resources are divided into two lists,
requested list and additional list, while the alignement of the additional
IOV BAR is not taken into the sizeing and assigning procedure.
This is reasonable in the original implementation, since IOV BAR's alignment is
mostly the size of a PF BAR alignemt. This means the alignment is already taken
into consideration. While this rule may be violated on some platform.
This patch take the additional IOV BAR alignment in sizing and assigning stage
explicitly.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
drivers/pci/setup-bus.c | 66 ++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 59 insertions(+), 7 deletions(-)
diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c
index 9509ffa..0c3b3a5 100644
--- a/drivers/pci/setup-bus.c
+++ b/drivers/pci/setup-bus.c
@@ -120,6 +120,28 @@ static resource_size_t get_res_add_size(struct list_head *head,
return 0;
}
+static resource_size_t get_res_add_align(struct list_head *head,
+ struct resource *res)
+{
+ struct pci_dev_resource *dev_res;
+
+ list_for_each_entry(dev_res, head, list) {
+ if (dev_res->res == res) {
+ int idx = res - &dev_res->dev->resource[0];
+
+ dev_printk(KERN_DEBUG, &dev_res->dev->dev,
+ "res[%d]=%pR get_res_add_align min_align %llx\n",
+ idx, dev_res->res,
+ (unsigned long long)dev_res->min_align);
+
+ return dev_res->min_align;
+ }
+ }
+
+ return 0;
+}
+
+
/* Sort resources by alignment */
static void pdev_sort_resources(struct pci_dev *dev, struct list_head *head)
{
@@ -369,8 +391,9 @@ static void __assign_resources_sorted(struct list_head *head,
LIST_HEAD(save_head);
LIST_HEAD(local_fail_head);
struct pci_dev_resource *save_res;
- struct pci_dev_resource *dev_res, *tmp_res;
+ struct pci_dev_resource *dev_res, *tmp_res, *dev_res2;
unsigned long fail_type;
+ resource_size_t add_align, align;
/* Check if optional add_size is there */
if (!realloc_head || list_empty(realloc_head))
@@ -385,10 +408,31 @@ static void __assign_resources_sorted(struct list_head *head,
}
/* Update res in head list with add_size in realloc_head list */
- list_for_each_entry(dev_res, head, list)
+ list_for_each_entry_safe(dev_res, tmp_res, head, list) {
dev_res->res->end += get_res_add_size(realloc_head,
dev_res->res);
+ if (!(dev_res->res->flags & IORESOURCE_STARTALIGN))
+ continue;
+
+ add_align = get_res_add_align(realloc_head, dev_res->res);
+
+ if (add_align > dev_res->res->start) {
+ dev_res->res->start = add_align;
+ dev_res->res->end = add_align +
+ resource_size(dev_res->res);
+
+ list_for_each_entry(dev_res2, head, list) {
+ align = pci_resource_alignment(dev_res2->dev,
+ dev_res2->res);
+ if (add_align > align)
+ list_move_tail(&dev_res->list,
+ &dev_res2->list);
+ }
+ }
+
+ }
+
/* Try updated head list with add_size added */
assign_requested_resources_sorted(head, &local_fail_head);
@@ -928,6 +972,8 @@ static int pbus_size_mem(struct pci_bus *bus, unsigned long mask,
mask | IORESOURCE_PREFETCH, type);
unsigned int mem64_mask = 0;
resource_size_t children_add_size = 0;
+ resource_size_t children_add_align = 0;
+ resource_size_t add_align = 0;
if (!b_res)
return 0;
@@ -955,6 +1001,7 @@ static int pbus_size_mem(struct pci_bus *bus, unsigned long mask,
/* put SRIOV requested res to the optional list */
if (realloc_head && i >= PCI_IOV_RESOURCES &&
i <= PCI_IOV_RESOURCE_END) {
+ add_align = max(pci_resource_alignment(dev, r), add_align);
r->end = r->start - 1;
add_to_list(realloc_head, dev, r, r_size, 0/* don't care */);
children_add_size += r_size;
@@ -982,8 +1029,11 @@ static int pbus_size_mem(struct pci_bus *bus, unsigned long mask,
max_order = order;
mem64_mask &= r->flags & IORESOURCE_MEM_64;
- if (realloc_head)
+ if (realloc_head) {
children_add_size += get_res_add_size(realloc_head, r);
+ children_add_align = get_res_add_align(realloc_head, r);
+ add_align = max(add_align, children_add_align);
+ }
}
}
@@ -994,7 +1044,7 @@ static int pbus_size_mem(struct pci_bus *bus, unsigned long mask,
add_size = children_add_size;
size1 = (!realloc_head || (realloc_head && !add_size)) ? size0 :
calculate_memsize(size, min_size, add_size,
- resource_size(b_res), min_align);
+ resource_size(b_res), max(min_align, add_align));
if (!size0 && !size1) {
if (b_res->start || b_res->end)
dev_info(&bus->self->dev, "disabling bridge window "
@@ -1007,10 +1057,12 @@ static int pbus_size_mem(struct pci_bus *bus, unsigned long mask,
b_res->end = size0 + min_align - 1;
b_res->flags |= IORESOURCE_STARTALIGN | mem64_mask;
if (size1 > size0 && realloc_head) {
- add_to_list(realloc_head, bus->self, b_res, size1-size0, min_align);
+ add_to_list(realloc_head, bus->self, b_res, size1-size0,
+ max(min_align, add_align));
dev_printk(KERN_DEBUG, &bus->self->dev, "bridge window "
- "%pR to %pR add_size %llx\n", b_res,
- &bus->busn_res, (unsigned long long)size1-size0);
+ "%pR to %pR add_size %llx add_align %llx\n", b_res,
+ &bus->busn_res, (unsigned long long)size1-size0,
+ max(min_align, add_align));
}
return 1;
}
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 12/17] powerpc/powernv: implement pcibios_sriov_resource_alignment on powernv
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
This patch implements the pcibios_sriov_resource_alignment() on powernv
platform.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/machdep.h | 1 +
arch/powerpc/kernel/pci-common.c | 8 ++++++++
arch/powerpc/platforms/powernv/pci-ioda.c | 17 +++++++++++++++++
3 files changed, 26 insertions(+)
diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h
index 2f2e770..3bbc55f 100644
--- a/arch/powerpc/include/asm/machdep.h
+++ b/arch/powerpc/include/asm/machdep.h
@@ -242,6 +242,7 @@ struct machdep_calls {
resource_size_t (*pcibios_window_alignment)(struct pci_bus *, unsigned long type);
#ifdef CONFIG_PCI_IOV
resource_size_t (*__pci_sriov_resource_size)(struct pci_dev *, int resno);
+ resource_size_t (*__pci_sriov_resource_alignment)(struct pci_dev *, int resno, resource_size_t align);
#endif /* CONFIG_PCI_IOV */
/* Called to shutdown machine specific hardware not already controlled
diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c
index c4e2e92..35345ac 100644
--- a/arch/powerpc/kernel/pci-common.c
+++ b/arch/powerpc/kernel/pci-common.c
@@ -128,6 +128,14 @@ resource_size_t pcibios_sriov_resource_size(struct pci_dev *pdev, int resno)
return 0;
}
+
+resource_size_t pcibios_sriov_resource_alignment(struct pci_dev *pdev, int resno, resource_size_t align)
+{
+ if (ppc_md.__pci_sriov_resource_alignment)
+ return ppc_md.__pci_sriov_resource_alignment(pdev, resno, align);
+
+ return 0;
+}
#endif /* CONFIG_PCI_IOV */
static resource_size_t pcibios_io_size(const struct pci_controller *hose)
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 7dfad6a..b0ac851 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1573,6 +1573,22 @@ static resource_size_t __pnv_pci_sriov_resource_size(struct pci_dev *pdev, int r
return size;
}
+
+static resource_size_t __pnv_pci_sriov_resource_alignment(struct pci_dev *pdev, int resno,
+ resource_size_t align)
+{
+ struct pci_dn *pdn = pci_get_pdn(pdev);
+ resource_size_t iov_align;
+
+ iov_align = resource_size(&pdev->resource[resno]);
+ if (iov_align)
+ return iov_align;
+
+ if (pdn->vfs)
+ return pdn->vfs * align;
+
+ return align;
+}
#endif /* CONFIG_PCI_IOV */
/* Prevent enabling devices for which we couldn't properly
@@ -1777,6 +1793,7 @@ void __init pnv_pci_init_ioda_phb(struct device_node *np,
ppc_md.pcibios_window_alignment = pnv_pci_window_alignment;
#ifdef CONFIG_PCI_IOV
ppc_md.__pci_sriov_resource_size = __pnv_pci_sriov_resource_size;
+ ppc_md.__pci_sriov_resource_alignment = __pnv_pci_sriov_resource_alignment;
#endif /* CONFIG_PCI_IOV */
pci_add_flags(PCI_REASSIGN_ALL_RSRC);
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 11/17] ppc/pnv: Expand VF resources according to the number of total_pe
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
On PHB3, VF resources will be covered by M64 BAR to have better PE isolation.
Mostly the total_pe number is different from the total_VFs, which will lead to
a conflict between MMIO space and the PE number.
This patch expands the VF resource size to reserve total_pe number of VFs'
resource, which prevents the conflict.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/machdep.h | 6 +++
arch/powerpc/include/asm/pci-bridge.h | 3 ++
arch/powerpc/kernel/pci-common.c | 15 ++++++
arch/powerpc/platforms/powernv/pci-ioda.c | 83 +++++++++++++++++++++++++++++
4 files changed, 107 insertions(+)
diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h
index ad3025d..2f2e770 100644
--- a/arch/powerpc/include/asm/machdep.h
+++ b/arch/powerpc/include/asm/machdep.h
@@ -234,9 +234,15 @@ struct machdep_calls {
/* Called after scan and before resource survey */
void (*pcibios_fixup_phb)(struct pci_controller *hose);
+#ifdef CONFIG_PCI_IOV
+ void (*pcibios_fixup_sriov)(struct pci_bus *bus);
+#endif /* CONFIG_PCI_IOV */
/* Called during PCI resource reassignment */
resource_size_t (*pcibios_window_alignment)(struct pci_bus *, unsigned long type);
+#ifdef CONFIG_PCI_IOV
+ resource_size_t (*__pci_sriov_resource_size)(struct pci_dev *, int resno);
+#endif /* CONFIG_PCI_IOV */
/* Called to shutdown machine specific hardware not already controlled
* by other drivers.
diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h
index 4ca90a3..8c849d8 100644
--- a/arch/powerpc/include/asm/pci-bridge.h
+++ b/arch/powerpc/include/asm/pci-bridge.h
@@ -168,6 +168,9 @@ struct pci_dn {
#define IODA_INVALID_PE (-1)
#ifdef CONFIG_PPC_POWERNV
int pe_number;
+#ifdef CONFIG_PCI_IOV
+ u16 vfs;
+#endif /* CONFIG_PCI_IOV */
#endif
};
diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c
index c449a26..c4e2e92 100644
--- a/arch/powerpc/kernel/pci-common.c
+++ b/arch/powerpc/kernel/pci-common.c
@@ -120,6 +120,16 @@ resource_size_t pcibios_window_alignment(struct pci_bus *bus,
return 1;
}
+#ifdef CONFIG_PCI_IOV
+resource_size_t pcibios_sriov_resource_size(struct pci_dev *pdev, int resno)
+{
+ if (ppc_md.__pci_sriov_resource_size)
+ return ppc_md.__pci_sriov_resource_size(pdev, resno);
+
+ return 0;
+}
+#endif /* CONFIG_PCI_IOV */
+
static resource_size_t pcibios_io_size(const struct pci_controller *hose)
{
#ifdef CONFIG_PPC64
@@ -1675,6 +1685,11 @@ void pcibios_scan_phb(struct pci_controller *hose)
if (ppc_md.pcibios_fixup_phb)
ppc_md.pcibios_fixup_phb(hose);
+#ifdef CONFIG_PCI_IOV
+ if (ppc_md.pcibios_fixup_sriov)
+ ppc_md.pcibios_fixup_sriov(bus);
+#endif /* CONFIG_PCI_IOV */
+
/* Configure PCI Express settings */
if (bus && !pci_has_flag(PCI_PROBE_ONLY)) {
struct pci_bus *child;
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 87cb3089..7dfad6a 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1298,6 +1298,67 @@ static void pnv_pci_init_ioda_msis(struct pnv_phb *phb)
static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { }
#endif /* CONFIG_PCI_MSI */
+#ifdef CONFIG_PCI_IOV
+static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
+{
+ struct pci_controller *hose;
+ struct pnv_phb *phb;
+ struct resource *res;
+ int i;
+ resource_size_t size;
+ struct pci_dn *pdn;
+
+ if (!pdev->is_physfn || pdev->is_added)
+ return;
+
+ hose = pci_bus_to_host(pdev->bus);
+ if (!hose) {
+ dev_err(&pdev->dev, "%s: NULL pci_controller\n", __func__);
+ return;
+ }
+
+ phb = hose->private_data;
+ if (!phb) {
+ dev_err(&pdev->dev, "%s: NULL PHB\n", __func__);
+ return;
+ }
+
+ pdn = pci_get_pdn(pdev);
+ pdn->vfs = 0;
+
+ for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
+ res = &pdev->resource[i];
+ if (!res->flags || res->parent)
+ continue;
+
+ if (!is_mem_pref_64_type(res->flags))
+ continue;
+
+ dev_info(&pdev->dev, "PowerNV: Fixing VF BAR[%d] %pR to\n",
+ i, res);
+ size = pci_sriov_resource_size(pdev, i);
+ res->end = res->start + size * phb->ioda.total_pe - 1;
+ dev_info(&pdev->dev, " %pR\n", res);
+ }
+ pdn->vfs = phb->ioda.total_pe;
+}
+
+static void pnv_pci_ioda_fixup_sriov(struct pci_bus *bus)
+{
+ struct pci_dev *pdev;
+ struct pci_bus *b;
+
+ list_for_each_entry(pdev, &bus->devices, bus_list) {
+ b = pdev->subordinate;
+
+ if (b)
+ pnv_pci_ioda_fixup_sriov(b);
+
+ pnv_pci_ioda_fixup_iov_resources(pdev);
+ }
+}
+#endif /* CONFIG_PCI_IOV */
+
/*
* This function is supposed to be called on basis of PE from top
* to bottom style. So the the I/O or MMIO segment assigned to
@@ -1498,6 +1559,22 @@ static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus,
return phb->ioda.io_segsize;
}
+#ifdef CONFIG_PCI_IOV
+static resource_size_t __pnv_pci_sriov_resource_size(struct pci_dev *pdev, int resno)
+{
+ struct pci_dn *pdn = pci_get_pdn(pdev);
+ u64 size = 0;
+
+ if (!pdn->vfs)
+ return size;
+
+ size = resource_size(pdev->resource + resno);
+ do_div(size, pdn->vfs);
+
+ return size;
+}
+#endif /* CONFIG_PCI_IOV */
+
/* Prevent enabling devices for which we couldn't properly
* assign a PE
*/
@@ -1692,9 +1769,15 @@ void __init pnv_pci_init_ioda_phb(struct device_node *np,
* for the P2P bridge bars so that each PCI bus (excluding
* the child P2P bridges) can form individual PE.
*/
+#ifdef CONFIG_PCI_IOV
+ ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_sriov;
+#endif /* CONFIG_PCI_IOV */
ppc_md.pcibios_fixup = pnv_pci_ioda_fixup;
ppc_md.pcibios_enable_device_hook = pnv_pci_enable_device_hook;
ppc_md.pcibios_window_alignment = pnv_pci_window_alignment;
+#ifdef CONFIG_PCI_IOV
+ ppc_md.__pci_sriov_resource_size = __pnv_pci_sriov_resource_size;
+#endif /* CONFIG_PCI_IOV */
pci_add_flags(PCI_REASSIGN_ALL_RSRC);
/* Reset IODA tables to a clean state */
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 13/17] powerpc/powernv: shift VF resource with an offset
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
On powrnv platform, resource position in M64 implies the PE# the resource
belongs to. In some particular case, adjustment of a resource is necessary to
locate it to a correct position in M64.
This patch introduce a function to shift the 'real' VF BAR address according to
an offset.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/platforms/powernv/pci-ioda.c | 30 +++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index b0ac851..e46c5bf 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -13,6 +13,7 @@
#include <linux/kernel.h>
#include <linux/pci.h>
+#include <linux/pci_regs.h>
#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/string.h>
@@ -544,6 +545,35 @@ static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev)
return 10;
}
+#ifdef CONFIG_PCI_IOV
+static void pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
+{
+ struct pci_dn *pdn = pci_get_pdn(dev);
+ int i;
+ struct resource *res;
+ resource_size_t size;
+
+ if (dev->is_physfn) {
+ for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
+ res = dev->resource + PCI_IOV_RESOURCES + i;
+ if (!res->flags || !res->parent)
+ continue;
+
+ if (!is_mem_pref_64_type(res->flags))
+ continue;
+
+ dev_info(&dev->dev, "PowerNV: Shifting VF BAR %pR to\n", res);
+ size = pci_sriov_resource_size(dev, PCI_IOV_RESOURCES + i);
+ res->start += size*offset;
+
+ dev_info(&dev->dev, " %pR\n", res);
+ pci_update_resource(dev, PCI_IOV_RESOURCES + i);
+ }
+ pdn->vfs -= offset;
+ }
+}
+#endif /* CONFIG_PCI_IOV */
+
#if 0
static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev)
{
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 14/17] ppc/pci: create/release dev-tree node for VFs
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
Currently, powernv platform is not aware of VFs. This means no dev-node
represents a VF. Also, VF PCI device is created when PF driver want to enable
it. This leads to the pdn->pdev and pdn->pe_number an invalid value.
This patch create/release dev-node for VF and fixs this when a VF's pci_dev
is created.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/platforms/powernv/Kconfig | 1 +
arch/powerpc/platforms/powernv/pci-ioda.c | 103 +++++++++++++++++++++++++++++
arch/powerpc/platforms/powernv/pci.c | 20 ++++++
3 files changed, 124 insertions(+)
diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig
index 895e8a2..0dd331b 100644
--- a/arch/powerpc/platforms/powernv/Kconfig
+++ b/arch/powerpc/platforms/powernv/Kconfig
@@ -11,6 +11,7 @@ config PPC_POWERNV
select PPC_UDBG_16550
select PPC_SCOM
select ARCH_RANDOM
+ select OF_DYNAMIC
default y
config PPC_POWERNV_RTAS
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index e46c5bf..9ace027 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -23,6 +23,7 @@
#include <linux/io.h>
#include <linux/msi.h>
#include <linux/memblock.h>
+#include <linux/of_pci.h>
#include <asm/sections.h>
#include <asm/io.h>
@@ -771,6 +772,108 @@ static void pnv_pci_ioda_setup_PEs(void)
}
}
+#ifdef CONFIG_PCI_IOV
+static void pnv_pci_create_vf_node(struct pci_dev *dev, u16 vf_num)
+{
+ struct device_node *dn, *p_dn;
+ struct pci_dn *pdn;
+ struct pci_controller *hose;
+ struct property *pp;
+ void* value;
+ u16 id;
+
+ hose = pci_bus_to_host(dev->bus);
+
+ /* Create dev-tree node for VFs if this is a PF */
+ p_dn = pci_bus_to_OF_node(dev->bus);
+ if (p_dn == NULL) {
+ dev_err(&dev->dev, "SRIOV: VF bus NULL device node\n");
+ return;
+ }
+
+ for (id = 0; id < vf_num; id++) {
+ dn = kzalloc(sizeof(*dn), GFP_KERNEL);
+ pdn = kzalloc(sizeof(*pdn), GFP_KERNEL);
+ pp = kzalloc(sizeof(*pp), GFP_KERNEL);
+ value = kzalloc(sizeof(u32), GFP_KERNEL);
+
+ if (!dn || !pdn || !pp || !value) {
+ kfree(dn);
+ kfree(pdn);
+ kfree(pp);
+ kfree(value);
+ dev_warn(&dev->dev, "%s: failed to create"
+ "dev-tree node for idx(%d)\n",
+ __func__, id);
+
+ break;
+ }
+
+ pp->value = value;
+ pdn->node = dn;
+ pdn->devfn = pci_iov_virtfn_devfn(dev, id);
+ pdn->busno = dev->bus->number;
+ pdn->pe_number = IODA_INVALID_PE;
+ pdn->phb = hose;
+
+ dn->data = pdn;
+ kref_init(&dn->kref);
+ dn->full_name = dn->name =
+ kasprintf(GFP_KERNEL, "%s/vf%d",
+ p_dn->full_name, pdn->devfn);
+ dn->parent = p_dn;
+
+ pp->name = kasprintf(GFP_KERNEL, "reg");
+ pp->length = 5 * sizeof(__be32);
+ *(u32*)pp->value = cpu_to_be32(pdn->devfn) << 8;
+ dn->properties = pp;
+
+ of_attach_node(dn);
+ }
+}
+
+static void pnv_pci_release_vf_node(struct pci_dev *dev, u16 vf_num)
+{
+ struct device_node *dn;
+ struct property *pp;
+ u16 id;
+
+ for (id = 0; id < vf_num; id++) {
+ dn = of_pci_find_child_device(dev->bus->dev.of_node,
+ pci_iov_virtfn_devfn(dev, id));
+ if (!dn)
+ continue;
+
+ of_detach_node(dn);
+ pp = dn->properties;
+ kfree(pp->name);
+ kfree(pp->value);
+ kfree(pp);
+ kfree(dn->data);
+ kfree(dn);
+ }
+}
+
+int pcibios_sriov_disable(struct pci_dev *pdev)
+{
+ struct pci_sriov *iov;
+ u16 vf_num;
+
+ iov = pdev->sriov;
+ vf_num = iov->num_VFs;
+ pnv_pci_release_vf_node(pdev, vf_num);
+
+ return 0;
+}
+
+int pcibios_sriov_enable(struct pci_dev *pdev, u16 vf_num)
+{
+ pnv_pci_create_vf_node(pdev, vf_num);
+
+ return 0;
+}
+#endif /* CONFIG_PCI_IOV */
+
static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev)
{
struct pci_dn *pdn = pci_get_pdn(pdev);
diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c
index 687a068..43fcc73 100644
--- a/arch/powerpc/platforms/powernv/pci.c
+++ b/arch/powerpc/platforms/powernv/pci.c
@@ -654,6 +654,26 @@ static void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
{
struct pci_controller *hose = pci_bus_to_host(pdev->bus);
struct pnv_phb *phb = hose->private_data;
+#ifdef CONFIG_PCI_IOV
+ struct pnv_ioda_pe *pe;
+ struct pci_dn *pdn;
+
+ /* Fix the VF pdn PE number */
+ if (pdev->is_virtfn) {
+ pdn = pci_get_pdn(pdev);
+ if (pdn->pcidev == NULL || pdn->pe_number == IODA_INVALID_PE) {
+ list_for_each_entry(pe, &phb->ioda.pe_list, list) {
+ if (pe->rid ==
+ ((pdev->bus->number << 8) | (pdev->devfn & 0xff))) {
+ pdn->pcidev = pdev;
+ pdn->pe_number = pe->pe_number;
+ pe->pdev = pdev;
+ break;
+ }
+ }
+ }
+ }
+#endif /* CONFIG_PCI_IOV */
/* If we have no phb structure, try to setup a fallback based on
* the device-tree (RTAS PCI for example)
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 16/17] ppc/pci: Expanding IOV BAR, with m64_per_iov supported
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
M64 aperture size is limited on PHB3. When the IOV BAR is too big, this will
exceed the limitation and failed to be assigned.
This patch introduce a different expanding based on the IOV BAR size:
IOV BAR size is smaller than 64M, expand to total_pe.
IOV BAR size is bigger than 64M, roundup power2.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/pci-bridge.h | 2 ++
arch/powerpc/platforms/powernv/pci-ioda.c | 28 ++++++++++++++++++++++++++--
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h
index 72f0af5..36b88e4 100644
--- a/arch/powerpc/include/asm/pci-bridge.h
+++ b/arch/powerpc/include/asm/pci-bridge.h
@@ -171,6 +171,8 @@ struct pci_dn {
#ifdef CONFIG_PCI_IOV
u16 vfs;
int offset;
+#define M64_PER_IOV 4
+ int m64_per_iov;
int m64_wins[PCI_SRIOV_NUM_BARS];
#endif /* CONFIG_PCI_IOV */
#endif
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index fb2c2c6..98fc163 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -1756,6 +1756,7 @@ static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
int i;
resource_size_t size;
struct pci_dn *pdn;
+ int mul, total_vfs;
if (!pdev->is_physfn || pdev->is_added)
return;
@@ -1775,6 +1776,10 @@ static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
pdn = pci_get_pdn(pdev);
pdn->vfs = 0;
+ total_vfs = pci_sriov_get_totalvfs(pdev);
+ pdn->m64_per_iov = 1;
+ mul = phb->ioda.total_pe;
+
for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
res = &pdev->resource[i];
if (!res->flags || res->parent)
@@ -1783,13 +1788,32 @@ static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
if (!is_mem_pref_64_type(res->flags))
continue;
+ size = pci_sriov_resource_size(pdev, i);
+
+ /* bigger than 64M */
+ if (size > (1 << 26)) {
+ dev_info(&pdev->dev, "PowerNV: VF BAR[%d] size "
+ "is bigger than 64M, roundup power2\n", i);
+ pdn->m64_per_iov = M64_PER_IOV;
+ mul = __roundup_pow_of_two(total_vfs);
+ break;
+ }
+ }
+
+ for (i = PCI_IOV_RESOURCES; i <= PCI_IOV_RESOURCE_END; i++) {
+ res = &pdev->resource[i];
+ if (!res->flags || res->parent)
+ continue;
+ if (!is_mem_pref_64_type(res->flags))
+ continue;
+
dev_info(&pdev->dev, "PowerNV: Fixing VF BAR[%d] %pR to\n",
i, res);
size = pci_sriov_resource_size(pdev, i);
- res->end = res->start + size * phb->ioda.total_pe - 1;
+ res->end = res->start + size * mul - 1;
dev_info(&pdev->dev, " %pR\n", res);
}
- pdn->vfs = phb->ioda.total_pe;
+ pdn->vfs = mul;
}
static void pnv_pci_ioda_fixup_sriov(struct pci_bus *bus)
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 15/17] powerpc/powernv: allocate VF PE
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
VFs are created, when driver intends to enable sriov.
This patch assign related resources and allocate PEs for VF at this moment.
This patch allocate enough M64 for IOV BAR and shift the VF resource to meet
the PE# indicated by M64.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/pci-bridge.h | 2 +
arch/powerpc/platforms/powernv/pci-ioda.c | 340 ++++++++++++++++++++++++++++-
arch/powerpc/platforms/powernv/pci.h | 10 +-
3 files changed, 339 insertions(+), 13 deletions(-)
diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h
index 8c849d8..72f0af5 100644
--- a/arch/powerpc/include/asm/pci-bridge.h
+++ b/arch/powerpc/include/asm/pci-bridge.h
@@ -170,6 +170,8 @@ struct pci_dn {
int pe_number;
#ifdef CONFIG_PCI_IOV
u16 vfs;
+ int offset;
+ int m64_wins[PCI_SRIOV_NUM_BARS];
#endif /* CONFIG_PCI_IOV */
#endif
};
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 9ace027..fb2c2c6 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -42,6 +42,17 @@
#include "powernv.h"
#include "pci.h"
+#ifdef CONFIG_PCI_IOV
+#define VF_PE_LOG \
+ else if (pe->flags & PNV_IODA_PE_VF) \
+ sprintf(pfix, "%04x:%02x:%2x.%d", \
+ pci_domain_nr(pe->parent_dev->bus), \
+ (pe->rid & 0xff00) >> 8, \
+ PCI_SLOT(pe->rid), PCI_FUNC(pe->rid));
+#else /* CONFIG_PCI_IOV*/
+#define VF_PE_LOG
+#endif /* CONFIG_PCI_IOV*/
+
#define define_pe_printk_level(func, kern_level) \
static int func(const struct pnv_ioda_pe *pe, const char *fmt, ...) \
{ \
@@ -55,13 +66,14 @@ static int func(const struct pnv_ioda_pe *pe, const char *fmt, ...) \
vaf.fmt = fmt; \
vaf.va = &args; \
\
- if (pe->pdev) \
+ if (pe->flags & PNV_IODA_PE_DEV) \
strlcpy(pfix, dev_name(&pe->pdev->dev), \
sizeof(pfix)); \
- else \
+ else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)) \
sprintf(pfix, "%04x:%02x ", \
pci_domain_nr(pe->pbus), \
pe->pbus->number); \
+ VF_PE_LOG \
r = printk(kern_level "pci %s: [PE# %.3d] %pV", \
pfix, pe->pe_number, &vaf); \
\
@@ -365,7 +377,12 @@ static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
}
rid_end = pe->rid + (count << 8);
}else {
- parent = pe->pdev->bus->self;
+#ifdef CONFIG_PCI_IOV
+ if (pe->flags & PNV_IODA_PE_VF)
+ parent = pe->parent_dev;
+ else
+#endif /* CONFIG_PCI_IOV */
+ parent = pe->pdev->bus->self;
bcomp = OpalPciBusAll;
dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
@@ -405,6 +422,9 @@ static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
pe->pbus = NULL;
pe->pdev = NULL;
+#ifdef CONFIG_PCI_IOV
+ pe->parent_dev = NULL;
+#endif /* CONFIG_PCI_IOV */
return 0;
}
@@ -443,7 +463,12 @@ static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe)
}
rid_end = pe->rid + (count << 8);
} else {
- parent = pe->pdev->bus->self;
+#ifdef CONFIG_PCI_IOV
+ if (pe->flags & PNV_IODA_PE_VF)
+ parent = pe->parent_dev;
+ else
+#endif /* CONFIG_PCI_IOV */
+ parent = pe->pdev->bus->self;
bcomp = OpalPciBusAll;
dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER;
fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER;
@@ -773,6 +798,114 @@ static void pnv_pci_ioda_setup_PEs(void)
}
#ifdef CONFIG_PCI_IOV
+static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
+{
+ struct pci_bus *bus;
+ struct pci_controller *hose;
+ struct pnv_phb *phb;
+ struct pci_dn *pdn;
+ int i;
+
+ bus = pdev->bus;
+ hose = pci_bus_to_host(bus);
+ phb = hose->private_data;
+ pdn = pci_get_pdn(pdev);
+
+ for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
+ if (pdn->m64_wins[i] == -1)
+ continue;
+ opal_pci_phb_mmio_enable(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i], 0);
+ clear_bit(pdn->m64_wins[i], &phb->ioda.m64win_alloc);
+ pdn->m64_wins[i] = -1;
+ }
+
+ return 0;
+}
+
+static int pnv_pci_vf_assign_m64(struct pci_dev *pdev)
+{
+ struct pci_bus *bus;
+ struct pci_controller *hose;
+ struct pnv_phb *phb;
+ struct pci_dn *pdn;
+ unsigned int win;
+ struct resource *res;
+ int i;
+ int64_t rc;
+
+ bus = pdev->bus;
+ hose = pci_bus_to_host(bus);
+ phb = hose->private_data;
+ pdn = pci_get_pdn(pdev);
+
+ /* Initialize the m64_wins to -1 */
+ for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
+ pdn->m64_wins[i] = -1;
+
+ for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
+ res = pdev->resource + PCI_IOV_RESOURCES + i;
+ if (!res->flags || !res->parent)
+ continue;
+
+ if (!is_mem_pref_64_type(res->flags))
+ continue;
+
+ do {
+ win = find_next_zero_bit(&phb->ioda.m64win_alloc,
+ phb->ioda.m64_bars, 0);
+
+ if (win >= phb->ioda.m64_bars)
+ goto m64_failed;
+ } while (test_and_set_bit(win, &phb->ioda.m64win_alloc));
+
+ pdn->m64_wins[i] = win;
+
+ /* Map the M64 here */
+ rc = opal_pci_set_phb_mem_window(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE,
+ pdn->m64_wins[i],
+ res->start,
+ 0, /* unused */
+ resource_size(res));
+ if (rc != OPAL_SUCCESS) {
+ pr_err("Failed to map M64 BAR #%d: %lld\n", win, rc);
+ goto m64_failed;
+ }
+
+ rc = opal_pci_phb_mmio_enable(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i], 1);
+ if (rc != OPAL_SUCCESS) {
+ pr_err("Failed to enable M64 BAR #%d: %llx\n", win, rc);
+ goto m64_failed;
+ }
+ }
+ return 0;
+
+m64_failed:
+ pnv_pci_vf_release_m64(pdev);
+ return -EBUSY;
+}
+
+static void pnv_pci_release_dev_dma(struct pci_dev *dev, struct pnv_ioda_pe *pe)
+{
+ struct pci_bus *bus;
+ struct pci_controller *hose;
+ struct pnv_phb *phb;
+ struct iommu_table *tbl;
+ unsigned long addr;
+
+ bus = dev->bus;
+ hose = pci_bus_to_host(bus);
+ phb = hose->private_data;
+ tbl = pe->tce32_table;
+ addr = tbl->it_base;
+
+ iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));
+ free_pages(addr, get_order(PNV_TCE32_TAB_SIZE));
+ pe->tce32_table = NULL;
+}
+
static void pnv_pci_create_vf_node(struct pci_dev *dev, u16 vf_num)
{
struct device_node *dn, *p_dn;
@@ -854,23 +987,186 @@ static void pnv_pci_release_vf_node(struct pci_dev *dev, u16 vf_num)
}
}
+static void pnv_ioda_release_vf_PE(struct pci_dev *pdev)
+{
+ struct pci_bus *bus;
+ struct pci_controller *hose;
+ struct pnv_phb *phb;
+ struct pnv_ioda_pe *pe, *pe_n;
+ struct pci_dn *pdn;
+
+ bus = pdev->bus;
+ hose = pci_bus_to_host(bus);
+ phb = hose->private_data;
+
+ if (!pdev->is_physfn)
+ return;
+
+ pdn = pci_get_pdn(pdev);
+ list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
+ if (pe->parent_dev != pdev)
+ continue;
+
+ pnv_pci_release_dev_dma(pdev, pe);
+
+ /* Remove from list */
+ mutex_lock(&phb->ioda.pe_list_mutex);
+ list_del(&pe->list);
+ mutex_unlock(&phb->ioda.pe_list_mutex);
+
+ pnv_ioda_deconfigure_pe(phb, pe);
+
+ pnv_ioda_free_pe(phb, pe->pe_number);
+ }
+}
+
int pcibios_sriov_disable(struct pci_dev *pdev)
{
- struct pci_sriov *iov;
+ struct pci_bus *bus;
+ struct pci_controller *hose;
+ struct pnv_phb *phb;
+ struct pci_dn *pdn;
+ struct pci_sriov *iov;
u16 vf_num;
+ bus = pdev->bus;
+ hose = pci_bus_to_host(bus);
+ phb = hose->private_data;
+ pdn = pci_get_pdn(pdev);
iov = pdev->sriov;
vf_num = iov->num_VFs;
+
+ /* Release VF PEs */
+ pnv_ioda_release_vf_PE(pdev);
pnv_pci_release_vf_node(pdev, vf_num);
+ if (phb->type == PNV_PHB_IODA2) {
+ pnv_pci_vf_resource_shift(pdev, -pdn->offset);
+
+ /* Release M64 BARs */
+ pnv_pci_vf_release_m64(pdev);
+
+ /* Release PE numbers */
+ bitmap_clear(phb->ioda.pe_alloc, pdn->offset, vf_num);
+ pdn->offset = 0;
+ }
+
return 0;
}
+static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
+ struct pnv_ioda_pe *pe);
+static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 vf_num)
+{
+ struct pci_bus *bus;
+ struct pci_controller *hose;
+ struct pnv_phb *phb;
+ struct pnv_ioda_pe *pe;
+ int pe_num;
+ u16 vf_index;
+ struct pci_dn *pdn;
+
+ bus = pdev->bus;
+ hose = pci_bus_to_host(bus);
+ phb = hose->private_data;
+ pdn = pci_get_pdn(pdev);
+
+ if (!pdev->is_physfn)
+ return;
+
+ /* Reserve PE for each VF */
+ for (vf_index = 0; vf_index < vf_num; vf_index++) {
+ pe_num = pdn->offset + vf_index;
+
+ pe = &phb->ioda.pe_array[pe_num];
+ pe->pe_number = pe_num;
+ pe->phb = phb;
+ pe->flags = PNV_IODA_PE_VF;
+ pe->pbus = NULL;
+ pe->parent_dev = pdev;
+ pe->tce32_seg = -1;
+ pe->mve_number = -1;
+ pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) |
+ pci_iov_virtfn_devfn(pdev, vf_index);
+
+ pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n",
+ hose->global_number, pdev->bus->number,
+ PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)),
+ PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num);
+
+ if (pnv_ioda_configure_pe(phb, pe)) {
+ /* XXX What do we do here ? */
+ if (pe_num)
+ pnv_ioda_free_pe(phb, pe_num);
+ pe->pdev = NULL;
+ continue;
+ }
+
+ pe->tce32_table = kzalloc_node(sizeof(struct iommu_table),
+ GFP_KERNEL, hose->node);
+ pe->tce32_table->data = pe;
+
+ /* Put PE to the list */
+ mutex_lock(&phb->ioda.pe_list_mutex);
+ list_add_tail(&pe->list, &phb->ioda.pe_list);
+ mutex_unlock(&phb->ioda.pe_list_mutex);
+
+ pnv_pci_ioda2_setup_dma_pe(phb, pe);
+
+ }
+}
+
int pcibios_sriov_enable(struct pci_dev *pdev, u16 vf_num)
{
+ struct pci_bus *bus;
+ struct pci_controller *hose;
+ struct pnv_phb *phb;
+ struct pci_dn *pdn;
+ int ret;
+
+ bus = pdev->bus;
+ hose = pci_bus_to_host(bus);
+ phb = hose->private_data;
+ pdn = pci_get_pdn(pdev);
+
+ if (phb->type == PNV_PHB_IODA2) {
+ /* Calculate available PE for required VFs */
+ mutex_lock(&phb->ioda.pe_alloc_mutex);
+ pdn->offset = bitmap_find_next_zero_area(
+ phb->ioda.pe_alloc, phb->ioda.total_pe,
+ 0, vf_num, 0);
+ if (pdn->offset >= phb->ioda.total_pe) {
+ mutex_unlock(&phb->ioda.pe_alloc_mutex);
+ pr_info("Failed to enable %d VFs, reduce VF number"
+ " and try again\n", vf_num);
+ pdn->offset = 0;
+ return -EBUSY;
+ }
+ bitmap_set(phb->ioda.pe_alloc, pdn->offset, vf_num);
+ mutex_unlock(&phb->ioda.pe_alloc_mutex);
+
+ /* Assign M64 BAR accordingly */
+ ret = pnv_pci_vf_assign_m64(pdev);
+ if (ret) {
+ pr_info("No enough M64 resource\n");
+ goto m64_failed;
+ }
+
+ /* Do some magic shift */
+ pnv_pci_vf_resource_shift(pdev, pdn->offset);
+ }
+
+ /* Setup VF PEs */
pnv_pci_create_vf_node(pdev, vf_num);
+ pnv_ioda_setup_vf_PE(pdev, vf_num);
return 0;
+
+m64_failed:
+ bitmap_clear(phb->ioda.pe_alloc, pdn->offset, vf_num);
+ pdn->offset = 0;
+
+ return ret;
}
#endif /* CONFIG_PCI_IOV */
@@ -1095,12 +1391,22 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb,
TCE_PCI_SWINV_PAIR;
}
iommu_init_table(tbl, phb->hose->node);
- iommu_register_group(tbl, pci_domain_nr(pe->pbus), pe->pe_number);
- if (pe->pdev)
+ if (pe->flags & PNV_IODA_PE_DEV) {
+ iommu_register_group(tbl, pci_domain_nr(pe->pdev->bus),
+ pe->pe_number);
set_iommu_table_base_and_group(&pe->pdev->dev, tbl);
- else
+ }
+ else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)) {
+ iommu_register_group(tbl, pci_domain_nr(pe->pbus),
+ pe->pe_number);
pnv_ioda_setup_bus_dma(pe, pe->pbus);
+ }
+#ifdef CONFIG_PCI_IOV
+ else if (pe->flags & PNV_IODA_PE_VF)
+ iommu_register_group(tbl, pci_domain_nr(pe->parent_dev->bus),
+ pe->pe_number);
+#endif /* CONFIG_PCI_IOV */
return;
fail:
@@ -1223,12 +1529,22 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb,
tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
}
iommu_init_table(tbl, phb->hose->node);
- iommu_register_group(tbl, pci_domain_nr(pe->pbus), pe->pe_number);
- if (pe->pdev)
+ if (pe->flags & PNV_IODA_PE_DEV) {
+ iommu_register_group(tbl, pci_domain_nr(pe->pdev->bus),
+ pe->pe_number);
set_iommu_table_base_and_group(&pe->pdev->dev, tbl);
- else
+ }
+ else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)) {
+ iommu_register_group(tbl, pci_domain_nr(pe->pbus),
+ pe->pe_number);
pnv_ioda_setup_bus_dma(pe, pe->pbus);
+ }
+#ifdef CONFIG_PCI_IOV
+ else if (pe->flags & PNV_IODA_PE_VF)
+ iommu_register_group(tbl, pci_domain_nr(pe->parent_dev->bus),
+ pe->pe_number);
+#endif /* CONFIG_PCI_IOV */
/* Also create a bypass window */
pnv_pci_ioda2_setup_bypass_pe(phb, pe);
@@ -1813,6 +2129,7 @@ void __init pnv_pci_init_ioda_phb(struct device_node *np,
phb->hub_id = hub_id;
phb->opal_id = phb_id;
phb->type = ioda_type;
+ mutex_init(&phb->ioda.pe_alloc_mutex);
/* Detect specific models for error handling */
if (of_device_is_compatible(np, "ibm,p7ioc-pciex"))
@@ -1873,6 +2190,7 @@ void __init pnv_pci_init_ioda_phb(struct device_node *np,
INIT_LIST_HEAD(&phb->ioda.pe_dma_list);
INIT_LIST_HEAD(&phb->ioda.pe_list);
+ mutex_init(&phb->ioda.pe_list_mutex);
/* Calculate how many 32-bit TCE segments we have */
phb->ioda.tce32_count = phb->ioda.m32_pci_base >> PNV_TCE32_SEG_SHIFT;
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index 9fbf7c0..e3ca524 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -28,8 +28,9 @@ static inline bool is_mem_pref_64_type(unsigned long flags)
#define PNV_PCI_DIAG_BUF_SIZE 8192
#define PNV_IODA_PE_DEV (1 << 0) /* PE has single PCI device */
-#define PNV_IODA_PE_BUS (1 << 1) /* PE has primary PCI bus */
-#define PNV_IODA_PE_BUS_ALL (1 << 2) /* PE has subordinate buses */
+#define PNV_IODA_PE_VF (1 << 1) /* PE for one VF */
+#define PNV_IODA_PE_BUS (1 << 2) /* PE has primary PCI bus */
+#define PNV_IODA_PE_BUS_ALL (1 << 3) /* PE has subordinate buses */
/* Data associated with a PE, including IOMMU tracking etc.. */
struct pnv_phb;
@@ -41,6 +42,9 @@ struct pnv_ioda_pe {
* entire bus (& children). In the former case, pdev
* is populated, in the later case, pbus is.
*/
+#ifdef CONFIG_PCI_IOV
+ struct pci_dev *parent_dev;
+#endif
struct pci_dev *pdev;
struct pci_bus *pbus;
@@ -156,6 +160,7 @@ struct pnv_phb {
/* PE allocation bitmap */
unsigned long *pe_alloc;
+ struct mutex pe_alloc_mutex;
/* M64 window allocation bitmap */
unsigned long m64win_alloc;
@@ -174,6 +179,7 @@ struct pnv_phb {
* on the sequence of creation
*/
struct list_head pe_list;
+ struct mutex pe_list_mutex;
/* Reverse map of PEs, will have to extend if
* we are to support more than 256 PEs, indexed
--
1.7.9.5
^ permalink raw reply related
* [RFC PATCH V3 17/17] ppc/pnv: Group VF PE when IOV BAR is big on PHB3
From: Wei Yang @ 2014-06-10 1:56 UTC (permalink / raw)
To: benh; +Cc: Wei Yang, linux-pci, gwshan, qiudayu, bhelgaas, yan, linuxppc-dev
In-Reply-To: <1402365399-5121-1-git-send-email-weiyang@linux.vnet.ibm.com>
When IOV BAR is big, each of it is covered by 4 M64 window. This leads to
several VF PE sits in one PE in terms of M64.
This patch group VF PEs according to the M64 allocation.
Signed-off-by: Wei Yang <weiyang@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/pci-bridge.h | 2 +-
arch/powerpc/platforms/powernv/pci-ioda.c | 183 +++++++++++++++++++++++------
2 files changed, 145 insertions(+), 40 deletions(-)
diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h
index 36b88e4..f0a21f5 100644
--- a/arch/powerpc/include/asm/pci-bridge.h
+++ b/arch/powerpc/include/asm/pci-bridge.h
@@ -173,7 +173,7 @@ struct pci_dn {
int offset;
#define M64_PER_IOV 4
int m64_per_iov;
- int m64_wins[PCI_SRIOV_NUM_BARS];
+ int m64_wins[PCI_SRIOV_NUM_BARS][M64_PER_IOV];
#endif /* CONFIG_PCI_IOV */
#endif
};
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c
index 98fc163..86688cd 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -804,26 +804,27 @@ static int pnv_pci_vf_release_m64(struct pci_dev *pdev)
struct pci_controller *hose;
struct pnv_phb *phb;
struct pci_dn *pdn;
- int i;
+ int i, j;
bus = pdev->bus;
hose = pci_bus_to_host(bus);
phb = hose->private_data;
pdn = pci_get_pdn(pdev);
- for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
- if (pdn->m64_wins[i] == -1)
- continue;
- opal_pci_phb_mmio_enable(phb->opal_id,
- OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i], 0);
- clear_bit(pdn->m64_wins[i], &phb->ioda.m64win_alloc);
- pdn->m64_wins[i] = -1;
- }
+ for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
+ for (j = 0; j < M64_PER_IOV; j++) {
+ if (pdn->m64_wins[i][j] == -1)
+ continue;
+ opal_pci_phb_mmio_enable(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0);
+ clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64win_alloc);
+ pdn->m64_wins[i][j] = -1;
+ }
return 0;
}
-static int pnv_pci_vf_assign_m64(struct pci_dev *pdev)
+static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 vf_num)
{
struct pci_bus *bus;
struct pci_controller *hose;
@@ -831,17 +832,33 @@ static int pnv_pci_vf_assign_m64(struct pci_dev *pdev)
struct pci_dn *pdn;
unsigned int win;
struct resource *res;
- int i;
+ int i, j;
int64_t rc;
+ int total_vfs;
+ resource_size_t size, start;
+ int pe_num;
+ int vf_groups;
+ int vf_per_group;
bus = pdev->bus;
hose = pci_bus_to_host(bus);
phb = hose->private_data;
pdn = pci_get_pdn(pdev);
+ total_vfs = pci_sriov_get_totalvfs(pdev);
/* Initialize the m64_wins to -1 */
for (i = 0; i < PCI_SRIOV_NUM_BARS; i++)
- pdn->m64_wins[i] = -1;
+ for (j = 0; j < 4; j++)
+ pdn->m64_wins[i][j] = -1;
+
+ if (pdn->m64_per_iov == M64_PER_IOV) {
+ vf_groups = (vf_num <= M64_PER_IOV) ? vf_num: M64_PER_IOV;
+ vf_per_group = (vf_num <= M64_PER_IOV)? 1:
+ __roundup_pow_of_two(vf_num) / pdn->m64_per_iov;
+ } else {
+ vf_groups = 1;
+ vf_per_group = 1;
+ }
for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
res = pdev->resource + PCI_IOV_RESOURCES + i;
@@ -851,33 +868,61 @@ static int pnv_pci_vf_assign_m64(struct pci_dev *pdev)
if (!is_mem_pref_64_type(res->flags))
continue;
- do {
- win = find_next_zero_bit(&phb->ioda.m64win_alloc,
- phb->ioda.m64_bars, 0);
-
- if (win >= phb->ioda.m64_bars)
- goto m64_failed;
- } while (test_and_set_bit(win, &phb->ioda.m64win_alloc));
+ for (j = 0; j < vf_groups; j++) {
+ do {
+ win = find_next_zero_bit(&phb->ioda.m64win_alloc,
+ phb->ioda.m64_bars, 0);
+
+ if (win >= phb->ioda.m64_bars)
+ goto m64_failed;
+ } while (test_and_set_bit(win, &phb->ioda.m64win_alloc));
+
+ pdn->m64_wins[i][j] = win;
+
+ if (pdn->m64_per_iov == M64_PER_IOV) {
+ size = pci_sriov_resource_size(pdev,
+ PCI_IOV_RESOURCES + i);
+ size = size * vf_per_group;
+ start = res->start + size * j;
+ } else {
+ size = resource_size(res);
+ start = res->start;
+ }
- pdn->m64_wins[i] = win;
+ /* Map the M64 here */
+ if (pdn->m64_per_iov == M64_PER_IOV) {
+ pe_num = pdn->offset + j;
+ rc = opal_pci_map_pe_mmio_window(phb->opal_id,
+ pe_num, OPAL_M64_WINDOW_TYPE,
+ pdn->m64_wins[i][j], 0);
+ }
- /* Map the M64 here */
- rc = opal_pci_set_phb_mem_window(phb->opal_id,
+ rc = opal_pci_set_phb_mem_window(phb->opal_id,
OPAL_M64_WINDOW_TYPE,
- pdn->m64_wins[i],
- res->start,
+ pdn->m64_wins[i][j],
+ start,
0, /* unused */
- resource_size(res));
- if (rc != OPAL_SUCCESS) {
- pr_err("Failed to map M64 BAR #%d: %lld\n", win, rc);
- goto m64_failed;
- }
+ size);
- rc = opal_pci_phb_mmio_enable(phb->opal_id,
- OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i], 1);
- if (rc != OPAL_SUCCESS) {
- pr_err("Failed to enable M64 BAR #%d: %llx\n", win, rc);
- goto m64_failed;
+
+ if (rc != OPAL_SUCCESS) {
+ pr_err("Failed to set M64 BAR #%d: %lld\n",
+ win, rc);
+ goto m64_failed;
+ }
+
+ if (pdn->m64_per_iov == M64_PER_IOV)
+ rc = opal_pci_phb_mmio_enable(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2);
+ else
+ rc = opal_pci_phb_mmio_enable(phb->opal_id,
+ OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1);
+
+ if (rc != OPAL_SUCCESS) {
+ pr_err("Failed to enable M64 BAR #%d: %llx\n",
+ win, rc);
+ goto m64_failed;
+ }
}
}
return 0;
@@ -987,21 +1032,51 @@ static void pnv_pci_release_vf_node(struct pci_dev *dev, u16 vf_num)
}
}
-static void pnv_ioda_release_vf_PE(struct pci_dev *pdev)
+static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 vf_num)
{
struct pci_bus *bus;
struct pci_controller *hose;
struct pnv_phb *phb;
struct pnv_ioda_pe *pe, *pe_n;
struct pci_dn *pdn;
+ u16 vf_index;
+ int64_t rc;
bus = pdev->bus;
hose = pci_bus_to_host(bus);
phb = hose->private_data;
+ pdn = pci_get_pdn(pdev);
if (!pdev->is_physfn)
return;
+ if (pdn->m64_per_iov == M64_PER_IOV && vf_num > M64_PER_IOV) {
+ int vf_group;
+ int vf_per_group;
+ int vf_index1;
+
+ vf_per_group = __roundup_pow_of_two(vf_num) / pdn->m64_per_iov;
+
+ for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
+ for (vf_index = vf_group * vf_per_group;
+ vf_index < (vf_group + 1) * vf_per_group;
+ vf_index++)
+ for (vf_index1 = vf_group * vf_per_group;
+ vf_index1 < (vf_group + 1) * vf_per_group;
+ vf_index1++){
+
+ rc = opal_pci_set_peltv(phb->opal_id,
+ pdn->offset + vf_index,
+ pdn->offset + vf_index1,
+ OPAL_REMOVE_PE_FROM_DOMAIN);
+
+ if (rc)
+ pr_warn("%s: Failed to unlink same"
+ " group PE#%d(%lld)\n", __func__,
+ pdn->offset + vf_index1, rc);
+ }
+ }
+
pdn = pci_get_pdn(pdev);
list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
if (pe->parent_dev != pdev)
@@ -1037,11 +1112,12 @@ int pcibios_sriov_disable(struct pci_dev *pdev)
vf_num = iov->num_VFs;
/* Release VF PEs */
- pnv_ioda_release_vf_PE(pdev);
+ pnv_ioda_release_vf_PE(pdev, vf_num);
pnv_pci_release_vf_node(pdev, vf_num);
if (phb->type == PNV_PHB_IODA2) {
- pnv_pci_vf_resource_shift(pdev, -pdn->offset);
+ if (pdn->m64_per_iov == 1)
+ pnv_pci_vf_resource_shift(pdev, -pdn->offset);
/* Release M64 BARs */
pnv_pci_vf_release_m64(pdev);
@@ -1065,6 +1141,7 @@ static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 vf_num)
int pe_num;
u16 vf_index;
struct pci_dn *pdn;
+ int64_t rc;
bus = pdev->bus;
hose = pci_bus_to_host(bus);
@@ -1112,7 +1189,34 @@ static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 vf_num)
mutex_unlock(&phb->ioda.pe_list_mutex);
pnv_pci_ioda2_setup_dma_pe(phb, pe);
+ }
+ if (pdn->m64_per_iov == M64_PER_IOV && vf_num > M64_PER_IOV) {
+ int vf_group;
+ int vf_per_group;
+ int vf_index1;
+
+ vf_per_group = __roundup_pow_of_two(vf_num) / pdn->m64_per_iov;
+
+ for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++)
+ for (vf_index = vf_group * vf_per_group;
+ vf_index < (vf_group + 1) * vf_per_group;
+ vf_index++)
+ for (vf_index1 = vf_group * vf_per_group;
+ vf_index1 < (vf_group + 1) * vf_per_group;
+ vf_index1++) {
+
+ rc = opal_pci_set_peltv(phb->opal_id,
+ pdn->offset + vf_index,
+ pdn->offset + vf_index1,
+ OPAL_ADD_PE_TO_DOMAIN);
+
+ if (rc)
+ pr_warn("%s: Failed to link same "
+ "group PE#%d(%lld)\n",
+ __func__,
+ pdn->offset + vf_index1, rc);
+ }
}
}
@@ -1146,14 +1250,15 @@ int pcibios_sriov_enable(struct pci_dev *pdev, u16 vf_num)
mutex_unlock(&phb->ioda.pe_alloc_mutex);
/* Assign M64 BAR accordingly */
- ret = pnv_pci_vf_assign_m64(pdev);
+ ret = pnv_pci_vf_assign_m64(pdev, vf_num);
if (ret) {
pr_info("No enough M64 resource\n");
goto m64_failed;
}
/* Do some magic shift */
- pnv_pci_vf_resource_shift(pdev, pdn->offset);
+ if (pdn->m64_per_iov == 1)
+ pnv_pci_vf_resource_shift(pdev, pdn->offset);
}
/* Setup VF PEs */
--
1.7.9.5
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox