[PATCH v19 23/26] vfio: introduce IOVA mode

[David Marchand <david.marchand@redhat.com>]

From: Anatoly Burakov <anatoly.burakov@intel.com>

Introduce dev_vfio_get_iova_mode() to return the IOVA remapping capability
(PA, VA or UNKNOWN).

Because the no-IOMMU check will rely on new mode infrastructure in
later commits, the VFIO initialization now needs to happen before bus
scan, as buses depend on no-IOMMU information.

Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
Signed-off-by: David Marchand <david.marchand@redhat.com>
---
 drivers/bus/fslmc/fslmc_bus.c          |   3 +-
 drivers/bus/fslmc/fslmc_vfio.c         |   2 +-
 drivers/bus/pci/linux/pci.c            |   2 +-
 drivers/net/hinic3/base/hinic3_hwdev.c |   3 +-
 drivers/net/ntnic/ntnic_ethdev.c       |   2 +-
 lib/eal/linux/eal.c                    |  15 ++-
 lib/eal/linux/eal_vfio.c               | 130 ++++++++++++++++++-------
 lib/eal/linux/eal_vfio.h               |   5 +
 lib/eal/linux/eal_vfio_mp_sync.c       |   5 +
 lib/eal/linux/include/dev_vfio.h       |  27 +++--
 10 files changed, 143 insertions(+), 51 deletions(-)

diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
index d55c414ead..dab3a8a677 100644
--- a/drivers/bus/fslmc/fslmc_bus.c
+++ b/drivers/bus/fslmc/fslmc_bus.c
@@ -500,7 +500,8 @@ rte_dpaa2_get_iommu_class(void)
 		return RTE_IOVA_DC;
 
 	/* check if all devices on the bus support Virtual addressing or not */
-	if (fslmc_all_device_support_iova() != 0 && dev_vfio_noiommu_is_enabled() == 0)
+	if (fslmc_all_device_support_iova() != 0 &&
+			dev_vfio_get_iova_mode() == DEV_VFIO_IOVA_MODE_VA)
 		return RTE_IOVA_VA;
 
 	return RTE_IOVA_PA;
diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
index c2b6ac1f3f..8e80336d51 100644
--- a/drivers/bus/fslmc/fslmc_vfio.c
+++ b/drivers/bus/fslmc/fslmc_vfio.c
@@ -204,7 +204,7 @@ fslmc_vfio_add_group(int vfio_group_fd,
 	group->fd = vfio_group_fd;
 	group->groupid = iommu_group_num;
 	rte_strscpy(group->group_name, group_name, sizeof(group->group_name));
-	if (dev_vfio_noiommu_is_enabled() > 0)
+	if (dev_vfio_get_iova_mode() == DEV_VFIO_IOVA_MODE_PA)
 		group->iommu_type = VFIO_NOIOMMU_IOMMU;
 	else
 		group->iommu_type = VFIO_TYPE1_IOMMU;
diff --git a/drivers/bus/pci/linux/pci.c b/drivers/bus/pci/linux/pci.c
index 832abe35b5..8594113323 100644
--- a/drivers/bus/pci/linux/pci.c
+++ b/drivers/bus/pci/linux/pci.c
@@ -631,7 +631,7 @@ pci_device_iova_mode(const struct rte_pci_driver *pdrv,
 		static int is_vfio_noiommu_enabled = -1;
 
 		if (is_vfio_noiommu_enabled == -1) {
-			if (dev_vfio_noiommu_is_enabled() == 1)
+			if (dev_vfio_get_iova_mode() == DEV_VFIO_IOVA_MODE_PA)
 				is_vfio_noiommu_enabled = 1;
 			else
 				is_vfio_noiommu_enabled = 0;
diff --git a/drivers/net/hinic3/base/hinic3_hwdev.c b/drivers/net/hinic3/base/hinic3_hwdev.c
index 6a6cf44752..9406b6cff2 100644
--- a/drivers/net/hinic3/base/hinic3_hwdev.c
+++ b/drivers/net/hinic3/base/hinic3_hwdev.c
@@ -78,7 +78,8 @@ hinic3_is_vfio_iommu_enable(const struct rte_eth_dev *eth_dev)
 {
 	struct rte_pci_device *pci_dev = RTE_CLASS_TO_BUS_DEVICE(eth_dev, *pci_dev);
 
-	return pci_dev->kdrv == RTE_PCI_KDRV_VFIO && dev_vfio_noiommu_is_enabled() != 1;
+	return pci_dev->kdrv == RTE_PCI_KDRV_VFIO &&
+			dev_vfio_get_iova_mode() == DEV_VFIO_IOVA_MODE_VA;
 }
 
 int
diff --git a/drivers/net/ntnic/ntnic_ethdev.c b/drivers/net/ntnic/ntnic_ethdev.c
index d79c4fc1d5..e0d84705f5 100644
--- a/drivers/net/ntnic/ntnic_ethdev.c
+++ b/drivers/net/ntnic/ntnic_ethdev.c
@@ -2729,7 +2729,7 @@ nthw_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
 			(pci_dev->device.devargs->data ? pci_dev->device.devargs->data : "NULL"));
 	}
 
-	const int n_rte_vfio_no_io_mmu_enabled = dev_vfio_noiommu_is_enabled();
+	const int n_rte_vfio_no_io_mmu_enabled = dev_vfio_get_iova_mode() == DEV_VFIO_IOVA_MODE_PA;
 	NT_LOG(DBG, NTNIC, "vfio_no_iommu_enabled=%d", n_rte_vfio_no_io_mmu_enabled);
 
 	if (n_rte_vfio_no_io_mmu_enabled) {
diff --git a/lib/eal/linux/eal.c b/lib/eal/linux/eal.c
index 419cebc236..a6003559da 100644
--- a/lib/eal/linux/eal.c
+++ b/lib/eal/linux/eal.c
@@ -674,6 +674,16 @@ rte_eal_init(int argc, char **argv)
 		}
 	}
 
+	/*
+	 * VFIO must be initialized before bus scan because buses need to know
+	 * about no-IOMMU mode status.
+	 */
+	if (dev_vfio_enable()) {
+		rte_eal_init_alert("Cannot init VFIO");
+		rte_errno = EAGAIN;
+		goto err_out;
+	}
+
 	if (rte_bus_scan()) {
 		rte_eal_init_alert("Cannot scan the buses for devices");
 		rte_errno = ENODEV;
@@ -769,11 +779,6 @@ rte_eal_init(int argc, char **argv)
 #endif
 	}
 
-	if (dev_vfio_enable()) {
-		rte_eal_init_alert("Cannot init VFIO");
-		rte_errno = EAGAIN;
-		goto err_out;
-	}
 	/* in secondary processes, memory init may allocate additional fbarrays
 	 * not present in primary processes, so to avoid any potential issues,
 	 * initialize memzones first.
diff --git a/lib/eal/linux/eal_vfio.c b/lib/eal/linux/eal_vfio.c
index fe60c57fc5..d2fe459774 100644
--- a/lib/eal/linux/eal_vfio.c
+++ b/lib/eal/linux/eal_vfio.c
@@ -40,6 +40,7 @@
 static struct vfio_container vfio_containers[RTE_MAX_VFIO_CONTAINERS];
 
 struct vfio_config vfio_global_cfg = {
+	.iova_mode = DEV_VFIO_IOVA_MODE_UNKNOWN,
 	.default_cfg = &vfio_containers[0]
 };
 
@@ -341,6 +342,36 @@ compact_user_maps(struct vfio_user_mem_maps *user_mem_maps)
 		user_mem_map_cmp);
 }
 
+#define VFIO_NOIOMMU_MODE_PATH "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode"
+
+static int
+vfio_noiommu_is_enabled(void)
+{
+	int fd;
+	ssize_t cnt;
+	char c;
+
+	fd = open(VFIO_NOIOMMU_MODE_PATH, O_RDONLY);
+	if (fd < 0) {
+		if (errno != ENOENT) {
+			EAL_LOG(ERR, "Cannot open VFIO noiommu file %i (%s)", errno,
+				strerror(errno));
+			return -1;
+		}
+		return 0;
+	}
+
+	cnt = read(fd, &c, 1);
+	close(fd);
+	if (cnt != 1) {
+		EAL_LOG(ERR, "Unable to read from VFIO noiommu file %i (%s)", errno,
+			strerror(errno));
+		return -1;
+	}
+
+	return c == 'Y';
+}
+
 static int
 vfio_open_group_fd(int iommu_group_num, bool mp_request)
 {
@@ -1072,6 +1103,46 @@ dev_vfio_release_device(const char *sysfs_base, const char *dev_addr,
 	return ret;
 }
 
+static int
+vfio_sync_iova_mode(enum dev_vfio_iova_mode *iova_mode)
+{
+	struct vfio_mp_param *p;
+	struct rte_mp_msg mp_req = {0};
+	struct rte_mp_reply mp_reply = {0};
+	struct timespec ts = {5, 0};
+
+	rte_strscpy(mp_req.name, EAL_VFIO_MP, sizeof(mp_req.name));
+	mp_req.len_param = sizeof(*p);
+	mp_req.num_fds = 0;
+	p = (struct vfio_mp_param *)mp_req.param;
+	p->req = VFIO_SOCKET_REQ_IOVA_MODE;
+
+	if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 && mp_reply.nb_received == 1) {
+		struct rte_mp_msg *mp_rep = &mp_reply.msgs[0];
+
+		p = (struct vfio_mp_param *)mp_rep->param;
+		if (p->result == VFIO_SOCKET_OK) {
+			*iova_mode = p->iova_mode;
+			free(mp_reply.msgs);
+			return 0;
+		}
+	}
+
+	free(mp_reply.msgs);
+	EAL_LOG(ERR, "Cannot request VFIO IOMMU mode");
+	return -1;
+}
+
+static const char *
+vfio_iova_mode_to_str(enum dev_vfio_iova_mode iova_mode)
+{
+	switch (iova_mode) {
+	case DEV_VFIO_IOVA_MODE_VA: return "VA";
+	case DEV_VFIO_IOVA_MODE_PA: return "PA";
+	default: return "unknown";
+	}
+}
+
 RTE_EXPORT_INTERNAL_SYMBOL(dev_vfio_enable)
 int
 dev_vfio_enable(void)
@@ -1141,8 +1212,25 @@ dev_vfio_enable(void)
 
 	/* check if we have VFIO driver enabled */
 	if (vfio_global_cfg.default_cfg->container_fd != -1) {
-		EAL_LOG(INFO, "VFIO support initialized");
 		vfio_enabled = true;
+
+		if (internal_conf->process_type == RTE_PROC_PRIMARY) {
+			int ret = vfio_noiommu_is_enabled();
+			if (ret < 0) {
+				EAL_LOG(ERR, "Cannot determine IOVA mode");
+				vfio_global_cfg.iova_mode = DEV_VFIO_IOVA_MODE_UNKNOWN;
+			} else if (ret == 1) {
+				vfio_global_cfg.iova_mode = DEV_VFIO_IOVA_MODE_PA;
+			} else {
+				vfio_global_cfg.iova_mode = DEV_VFIO_IOVA_MODE_VA;
+			}
+		} else {
+			if (vfio_sync_iova_mode(&vfio_global_cfg.iova_mode) < 0)
+				vfio_global_cfg.iova_mode = DEV_VFIO_IOVA_MODE_UNKNOWN;
+		}
+
+		EAL_LOG(NOTICE, "VFIO support initialized: IOVA as %s",
+			vfio_iova_mode_to_str(vfio_global_cfg.iova_mode));
 	} else {
 		EAL_LOG(NOTICE, "VFIO support could not be initialized");
 	}
@@ -1522,39 +1610,6 @@ container_dma_unmap(struct vfio_container *cfg, uint64_t vaddr, uint64_t iova, u
 	return ret;
 }
 
-RTE_EXPORT_INTERNAL_SYMBOL(dev_vfio_noiommu_is_enabled)
-int
-dev_vfio_noiommu_is_enabled(void)
-{
-	int fd;
-	ssize_t cnt;
-	char c;
-
-	fd = open(DEV_VFIO_NOIOMMU_MODE, O_RDONLY);
-	if (fd < 0) {
-		if (errno != ENOENT) {
-			EAL_LOG(ERR, "Cannot open VFIO noiommu file "
-					"%i (%s)", errno, strerror(errno));
-			return -1;
-		}
-		/*
-		 * else the file does not exists
-		 * i.e. noiommu is not enabled
-		 */
-		return 0;
-	}
-
-	cnt = read(fd, &c, 1);
-	close(fd);
-	if (cnt != 1) {
-		EAL_LOG(ERR, "Unable to read from VFIO noiommu file "
-				"%i (%s)", errno, strerror(errno));
-		return -1;
-	}
-
-	return c == 'Y';
-}
-
 RTE_EXPORT_INTERNAL_SYMBOL(dev_vfio_container_create)
 int
 dev_vfio_container_create(void)
@@ -1802,6 +1857,13 @@ vfio_cleanup_config(struct vfio_container *cfg)
 	return 0;
 }
 
+RTE_EXPORT_INTERNAL_SYMBOL(dev_vfio_get_iova_mode)
+enum dev_vfio_iova_mode
+dev_vfio_get_iova_mode(void)
+{
+	return vfio_global_cfg.iova_mode;
+}
+
 RTE_EXPORT_INTERNAL_SYMBOL(dev_vfio_cleanup)
 void
 dev_vfio_cleanup(void)
diff --git a/lib/eal/linux/eal_vfio.h b/lib/eal/linux/eal_vfio.h
index 5dae09b125..ce81526daf 100644
--- a/lib/eal/linux/eal_vfio.h
+++ b/lib/eal/linux/eal_vfio.h
@@ -9,6 +9,8 @@
 
 #include <stdint.h>
 
+#include <dev_vfio.h>
+
 /* hot plug/unplug of VFIO groups may cause all DMA maps to be dropped. we can
  * recreate the mappings for DPDK segments, but we cannot do so for memory that
  * was registered by the user themselves, so we need to store the user mappings
@@ -78,6 +80,7 @@ int vfio_open_container_fd(bool mp_request);
 /* global configuration */
 struct vfio_config {
 	struct vfio_container *default_cfg;
+	enum dev_vfio_iova_mode iova_mode;
 	const struct vfio_iommu_ops *ops;
 };
 
@@ -105,6 +108,7 @@ void vfio_mp_sync_cleanup(void);
 #define VFIO_SOCKET_REQ_CONTAINER 0x100
 #define VFIO_SOCKET_REQ_GROUP 0x200
 #define VFIO_SOCKET_REQ_IOMMU_TYPE 0x400
+#define VFIO_SOCKET_REQ_IOVA_MODE 0x800
 #define VFIO_SOCKET_OK 0x0
 #define VFIO_SOCKET_NO_FD 0x1
 #define VFIO_SOCKET_ERR 0xFF
@@ -115,6 +119,7 @@ struct vfio_mp_param {
 	union {
 		int group_num;
 		int iommu_type_id;
+		enum dev_vfio_iova_mode iova_mode;
 	};
 };
 
diff --git a/lib/eal/linux/eal_vfio_mp_sync.c b/lib/eal/linux/eal_vfio_mp_sync.c
index f394f3d227..aa90ab7bd0 100644
--- a/lib/eal/linux/eal_vfio_mp_sync.c
+++ b/lib/eal/linux/eal_vfio_mp_sync.c
@@ -74,6 +74,11 @@ vfio_mp_primary(const struct rte_mp_msg *msg, const void *peer)
 		}
 		break;
 	}
+	case VFIO_SOCKET_REQ_IOVA_MODE:
+		r->req = VFIO_SOCKET_REQ_IOVA_MODE;
+		r->iova_mode = vfio_global_cfg.iova_mode;
+		r->result = VFIO_SOCKET_OK;
+		break;
 	default:
 		EAL_LOG(ERR, "vfio received invalid message!");
 		return -1;
diff --git a/lib/eal/linux/include/dev_vfio.h b/lib/eal/linux/include/dev_vfio.h
index 4e2c21ae89..f3ce665366 100644
--- a/lib/eal/linux/include/dev_vfio.h
+++ b/lib/eal/linux/include/dev_vfio.h
@@ -27,8 +27,6 @@ extern "C" {
 #define DEV_VFIO_CONTAINER_PATH "/dev/vfio/vfio"
 #define DEV_VFIO_GROUP_FMT "/dev/vfio/%u"
 #define DEV_VFIO_NOIOMMU_GROUP_FMT "/dev/vfio/noiommu-%u"
-#define DEV_VFIO_NOIOMMU_MODE      \
-	"/sys/module/vfio/parameters/enable_unsafe_noiommu_mode"
 
 /* we don't need an actual definition, only pointer is used */
 struct vfio_device_info;
@@ -46,6 +44,22 @@ enum dev_vfio_module {
 	DEV_VFIO_MODULE_VFIO_PCI, /**< VFIO PCI module. */
 };
 
+/**
+ * @enum dev_vfio_iova_mode
+ * IOVA modes.
+ *
+ * These modes describe IOVA remapping capability.
+ *
+ * - DEV_VFIO_IOVA_MODE_UNKNOWN: IOVA mode is unknown.
+ * - DEV_VFIO_IOVA_MODE_VA: IOVA addresses can be remapped.
+ * - DEV_VFIO_IOVA_MODE_PA: IOVA addresses cannot be remapped.
+ */
+enum dev_vfio_iova_mode {
+	DEV_VFIO_IOVA_MODE_UNKNOWN = 0, /**< IOVA mode not determined */
+	DEV_VFIO_IOVA_MODE_VA,          /**< IOVA addresses can be remapped */
+	DEV_VFIO_IOVA_MODE_PA,          /**< IOVA addresses cannot be remapped */
+};
+
 /**
  * @internal
  * Set up a device managed by VFIO driver.
@@ -136,15 +150,14 @@ int dev_vfio_is_enabled(void);
 
 /**
  * @internal
- * Whether VFIO NOIOMMU mode is enabled.
+ * Get current VFIO IOVA mode.
  *
  * @return
- *   1 if true.
- *   0 if false.
- *   <0 for errors.
+ *   VFIO IOVA mode currently in use.
  */
 __rte_internal
-int dev_vfio_noiommu_is_enabled(void);
+enum dev_vfio_iova_mode
+dev_vfio_get_iova_mode(void);
 
 /**
  * @internal
-- 
2.54.0