From mboxrd@z Thu Jan 1 00:00:00 1970 From: Hemant Agrawal Subject: Re: [PATCH v4 05/12] linuxapp/eal_pci: get iommu class Date: Tue, 18 Jul 2017 16:25:21 +0530 Message-ID: <12f596f3-b5c2-46e4-63e3-85e7a9ec75f1@nxp.com> References: <20170711061631.5018-1-santosh.shukla@caviumnetworks.com> <20170718055950.10208-1-santosh.shukla@caviumnetworks.com> <20170718055950.10208-6-santosh.shukla@caviumnetworks.com> Mime-Version: 1.0 Content-Type: text/plain; charset="windows-1252"; format=flowed Content-Transfer-Encoding: 7bit Cc: , , , , , , , , To: Santosh Shukla , , Return-path: Received: from NAM02-BL2-obe.outbound.protection.outlook.com (mail-bl2nam02on0085.outbound.protection.outlook.com [104.47.38.85]) by dpdk.org (Postfix) with ESMTP id 4C46F3253 for ; Tue, 18 Jul 2017 12:55:36 +0200 (CEST) In-Reply-To: <20170718055950.10208-6-santosh.shukla@caviumnetworks.com> List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" On 7/18/2017 11:29 AM, Santosh Shukla wrote: > Get iommu class of PCI device on the bus and returns preferred iova > mapping mode for that bus. > > Algorithm for iova scheme selection for PCI bus: > 0. If no device bound then return with RTE_IOVA_DC mapping mode, > else goto 1). > 1. Look for device attached to vfio kdrv and has .drv_flag set > to RTE_PCI_DRV_IOVA_AS_VA. > 2. Look for any device attached to UIO class of driver. > 3. Check for vfio-noiommu mode enabled. > > If 2) & 3) is false and 1) is true then select > mapping scheme as RTE_IOVA_VA. Otherwise use default > mapping scheme (RTE_IOVA_PA). > > Signed-off-by: Santosh Shukla > Signed-off-by: Jerin Jacob > --- > v3 --> v4 : > - Reworded WARNING message (suggested by Maxime) > - Added pci_device_is_bound func to check for no device case > (suggested by Hemant). > - Added ifdef vfio_present. > > v1 --> v2: > - Removed Linux version check in vfio_noiommu func. Refer [1]. > - Extending autodetction logic for _iommu_class. > Refer [2]. > > [1] https://www.mail-archive.com/dev@dpdk.org/msg70108.html > [2] https://www.mail-archive.com/dev@dpdk.org/msg70279.html > > lib/librte_eal/linuxapp/eal/eal_pci.c | 95 +++++++++++++++++++++++++ > lib/librte_eal/linuxapp/eal/eal_vfio.c | 19 +++++ > lib/librte_eal/linuxapp/eal/eal_vfio.h | 4 ++ > lib/librte_eal/linuxapp/eal/rte_eal_version.map | 1 + > 4 files changed, 119 insertions(+) > > diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c > index 7d9e1a99b..ecd946250 100644 > --- a/lib/librte_eal/linuxapp/eal/eal_pci.c > +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c > @@ -45,6 +45,7 @@ > #include "eal_filesystem.h" > #include "eal_private.h" > #include "eal_pci_init.h" > +#include "eal_vfio.h" > > /** > * @file > @@ -488,6 +489,100 @@ rte_pci_scan(void) > return -1; > } > > +/* > + * Is pci device bound to any kdrv > + */ > +static inline int > +pci_device_is_bound(void) > +{ > + struct rte_pci_device *dev = NULL; > + int ret = 0; > + > + FOREACH_DEVICE_ON_PCIBUS(dev) { > + if (dev->kdrv == RTE_KDRV_UNKNOWN || > + dev->kdrv == RTE_KDRV_NONE) { > + continue; > + } else { > + ret = 1; > + break; > + } > + } > + return ret; > +} > + > +/* > + * Any one of the device bound to uio > + */ > +static inline int > +pci_device_bound_uio(void) > +{ > + struct rte_pci_device *dev = NULL; > + > + FOREACH_DEVICE_ON_PCIBUS(dev) { > + if (dev->kdrv == RTE_KDRV_IGB_UIO || > + dev->kdrv == RTE_KDRV_UIO_GENERIC) { > + return 1; > + } > + } > + return 0; > +} > + > +/* > + * Any one of the device has iova as va > + */ > +static inline int > +pci_device_has_iova_va(void) > +{ > + struct rte_pci_device *dev = NULL; > + struct rte_pci_driver *drv = NULL; > + > + FOREACH_DRIVER_ON_PCIBUS(drv) { > + if (drv && drv->drv_flags & RTE_PCI_DRV_IOVA_AS_VA) { > + FOREACH_DEVICE_ON_PCIBUS(dev) { > + if (dev->kdrv == RTE_KDRV_VFIO && > + rte_pci_match(drv, dev)) > + return 1; > + } > + } > + } > + return 0; > +} > + > +/* > + * Get iommu class of PCI devices on the bus. > + */ > +enum rte_iova_mode > +rte_pci_get_iommu_class(void) > +{ > + bool is_bound; > + bool is_vfio_noiommu_enabled = true; > + bool has_iova_va; > + bool is_bound_uio; > + > + is_bound = pci_device_is_bound(); > + if (!is_bound) > + return RTE_IOVA_DC; > + > + has_iova_va = pci_device_has_iova_va(); > + is_bound_uio = pci_device_bound_uio(); > +#ifdef VFIO_PRESENT > + is_vfio_noiommu_enabled = vfio_noiommu_is_enabled() == 1 ? 1 : 0; > +#endif > + > + if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled) > + return RTE_IOVA_VA; > + > + if (has_iova_va) { > + RTE_LOG(WARNING, EAL, "Some devices want iova as va but pa will be used because.. "); > + if (is_vfio_noiommu_enabled) > + RTE_LOG(WARNING, EAL, "vfio-noiommu mode configured\n"); > + if (is_bound_uio) > + RTE_LOG(WARNING, EAL, "few device bound to UIO\n"); > + } > + > + return RTE_IOVA_PA; > +} > + > /* Read PCI config space. */ > int rte_pci_read_config(const struct rte_pci_device *device, > void *buf, size_t len, off_t offset) > diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.c b/lib/librte_eal/linuxapp/eal/eal_vfio.c > index 946df7e31..c8a97b7e7 100644 > --- a/lib/librte_eal/linuxapp/eal/eal_vfio.c > +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.c > @@ -816,4 +816,23 @@ vfio_noiommu_dma_map(int __rte_unused vfio_container_fd) > return 0; > } > > +int > +vfio_noiommu_is_enabled(void) > +{ > + int fd, ret, cnt __rte_unused; > + char c; > + > + ret = -1; > + fd = open(VFIO_NOIOMMU_MODE, O_RDONLY); > + if (fd < 0) > + return -1; > + > + cnt = read(fd, &c, 1); > + if (c == 'Y') > + ret = 1; > + > + close(fd); > + return ret; > +} > + > #endif > diff --git a/lib/librte_eal/linuxapp/eal/eal_vfio.h b/lib/librte_eal/linuxapp/eal/eal_vfio.h > index 5ff63e5d7..26ea8e119 100644 > --- a/lib/librte_eal/linuxapp/eal/eal_vfio.h > +++ b/lib/librte_eal/linuxapp/eal/eal_vfio.h > @@ -150,6 +150,8 @@ struct vfio_config { > #define VFIO_NOIOMMU_GROUP_FMT "/dev/vfio/noiommu-%u" > #define VFIO_GET_REGION_ADDR(x) ((uint64_t) x << 40ULL) > #define VFIO_GET_REGION_IDX(x) (x >> 40) > +#define VFIO_NOIOMMU_MODE \ > + "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode" > > /* DMA mapping function prototype. > * Takes VFIO container fd as a parameter. > @@ -210,6 +212,8 @@ int pci_vfio_is_enabled(void); > > int vfio_mp_sync_setup(void); > > +int vfio_noiommu_is_enabled(void); > + > #define SOCKET_REQ_CONTAINER 0x100 > #define SOCKET_REQ_GROUP 0x200 > #define SOCKET_CLR_GROUP 0x300 > diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map > index a69bbb599..5dd40f948 100644 > --- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map > +++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map > @@ -206,6 +206,7 @@ DPDK_17.08 { > rte_bus_find_by_device; > rte_bus_find_by_name; > rte_pci_match; > + rte_pci_get_iommu_class; > > } DPDK_17.05; > > Acked-by: Hemant Agrawal