[PATCH v2 1/2] app/testpmd: add test for remote PD and CTX

[Michael Baum <michaelba@nvidia.com>]

Add mlx5 internal option in testpmd run-time function "port attach" to
add another parameter named "mlx5_socket" for attaching port and add 2
devargs before.

The arguments are "cmd_fd" and "pd_handle" using to import device
created out of PMD. Testpmd application import it using IPC, and updates
the devargs list before attaching.

The syntax is:

  testpmd > port attach (identifier) mlx5_socket=(path)

Where "path" is the IPC socket path agreed on the remote process.

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
---
 app/test-pmd/cmdline.c          |  14 ++-
 app/test-pmd/testpmd.c          |   5 ++
 doc/guides/nics/mlx5.rst        |  44 ++++++++++
 drivers/net/mlx5/mlx5_testpmd.c | 145 ++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_testpmd.h |  16 ++++
 5 files changed, 222 insertions(+), 2 deletions(-)

diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index a59e6166d5..869ecd3d2a 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -780,6 +780,12 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"port attach (ident)\n"
 			"    Attach physical or virtual dev by pci address or virtual device name\n\n"
 
+#ifdef RTE_NET_MLX5
+			"port attach (ident) mlx5_socket=(path)\n"
+			"    Attach physical or virtual dev by pci address or virtual device name "
+			"and add \"cmd_fd\" and \"pd_handle\" devargs before attaching\n\n"
+#endif
+
 			"port detach (port_id)\n"
 			"    Detach physical or virtual dev by port_id\n\n"
 
@@ -1401,8 +1407,12 @@ static cmdline_parse_token_string_t cmd_operate_attach_port_identifier =
 static cmdline_parse_inst_t cmd_operate_attach_port = {
 	.f = cmd_operate_attach_port_parsed,
 	.data = NULL,
-	.help_str = "port attach <identifier>: "
-		"(identifier: pci address or virtual dev name)",
+	.help_str = "port attach <identifier> mlx5_socket=<path>: "
+		"(identifier: pci address or virtual dev name"
+#ifdef RTE_NET_MLX5
+		", path (optional): socket path to get cmd FD and PD handle"
+#endif
+		")",
 	.tokens = {
 		(void *)&cmd_operate_attach_port_port,
 		(void *)&cmd_operate_attach_port_keyword,
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index e6321bdedb..d2df6732a0 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -3360,6 +3360,11 @@ attach_port(char *identifier)
 		return;
 	}
 
+#if defined(RTE_NET_MLX5) && !defined(RTE_EXEC_ENV_WINDOWS)
+	if (mlx5_test_attach_port_extend_devargs(identifier) < 0)
+		return;
+#endif
+
 	if (rte_dev_probe(identifier) < 0) {
 		TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier);
 		return;
diff --git a/doc/guides/nics/mlx5.rst b/doc/guides/nics/mlx5.rst
index 1b66b2bc33..392292cc95 100644
--- a/doc/guides/nics/mlx5.rst
+++ b/doc/guides/nics/mlx5.rst
@@ -1777,3 +1777,47 @@ the command sets the current shaper to 5Gbps and disables avail_thresh_triggered
 .. code-block:: console
 
    testpmd> mlx5 set port 1 host_shaper avail_thresh_triggered 0 rate 50
+
+
+Testpmd
+-------
+
+port attach with socket path
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Attach a port specified by pci address or virtual device args and add extra
+devargs to it, which is imported from external process::
+
+   testpmd> port attach (identifier) mlx5_socket=(path)
+
+where:
+
+* ``identifier``: pci address or virtual device args.
+* ``path``: socket path to import arguments agreed by the external process.
+
+The mlx5 PMD enables to import CTX and PD created outside the PMD.
+It gets as devargs the device's ``cmd_fd`` and ``pd_handle``,
+then using those arguments to import objects.
+See :ref:`mlx5 driver options <mlx5_common_driver_options>` for more information.
+
+When ``cmd_fd`` and ``pd_handle`` arguments are coming from another process,
+the FD must be dup'd before being passed.
+In this function, testpmd initializes IPC socket to get FD using SCM_RIGHTS.
+It gets the external process socket path, then import the ``cmd_fd`` and
+``pd_handle`` arguments and add them to devargs list.
+After updating this, it calls the regular ``port attach`` function
+with extended identifier.
+
+For example, to attach a port whose pci address is ``0000:0a:00.0`` and its
+socket path is ``/var/run/import_ipc_socket``.
+
+.. code-block:: console
+
+   testpmd> port attach 0000:0a:00.0 mlx5_socket=/var/run/import_ipc_socket
+   Attaching a new port...
+   testpmd: MLX5 socket path is /var/run/import_ipc_socket
+   testpmd: Attach port with extra devargs 0000:0a:00.0,cmd_fd=40,pd_handle=1
+   EAL: Probe PCI driver: mlx5_pci (15b3:101d) device: 0000:0a:00.0 (socket 0)
+   Port 0 is attached. Now total ports is 1
+   Done
+
diff --git a/drivers/net/mlx5/mlx5_testpmd.c b/drivers/net/mlx5/mlx5_testpmd.c
index 98bd395ae0..46444f06e6 100644
--- a/drivers/net/mlx5/mlx5_testpmd.c
+++ b/drivers/net/mlx5/mlx5_testpmd.c
@@ -6,6 +6,11 @@
 #include <stdint.h>
 #include <string.h>
 #include <stdlib.h>
+#include <unistd.h>
+#ifndef RTE_EXEC_ENV_WINDOWS
+#include <sys/socket.h>
+#include <sys/un.h>
+#endif
 
 #include <rte_prefetch.h>
 #include <rte_common.h>
@@ -14,6 +19,7 @@
 #include <rte_alarm.h>
 #include <rte_pmd_mlx5.h>
 #include <rte_ethdev.h>
+
 #include "mlx5_testpmd.h"
 #include "testpmd.h"
 
@@ -111,6 +117,145 @@ mlx5_test_set_port_host_shaper(uint16_t port_id, uint16_t avail_thresh_triggered
 	return 0;
 }
 
+#ifndef RTE_EXEC_ENV_WINDOWS
+static const char*
+mlx5_test_get_socket_path(char *extend)
+{
+	if (strstr(extend, "mlx5_socket=") == extend) {
+		const char *socket_path = strchr(extend, '=') + 1;
+
+		TESTPMD_LOG(DEBUG, "MLX5 socket path is %s\n", socket_path);
+		return socket_path;
+	}
+
+	TESTPMD_LOG(ERR, "Failed to extract a valid socket path from %s\n",
+		    extend);
+	return NULL;
+}
+
+static int
+mlx5_test_extend_devargs(char *identifier, char *extend)
+{
+	struct sockaddr_un un = {
+		.sun_family = AF_UNIX,
+	};
+	int cmd_fd;
+	int pd_handle;
+	struct iovec iov = {
+		.iov_base = &pd_handle,
+		.iov_len = sizeof(int),
+	};
+	union {
+		char buf[CMSG_SPACE(sizeof(int))];
+		struct cmsghdr align;
+	} control;
+	struct msghdr msgh = {
+		.msg_iov = NULL,
+		.msg_iovlen = 0,
+	};
+	struct cmsghdr *cmsg;
+	const char *path = mlx5_test_get_socket_path(extend + 1);
+	size_t len = 1;
+	int socket_fd;
+	int ret;
+
+	if (path == NULL) {
+		TESTPMD_LOG(ERR, "Invalid devargs extension is specified\n");
+		return -1;
+	}
+
+	/* Initialize IPC channel. */
+	socket_fd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
+	if (socket_fd < 0) {
+		TESTPMD_LOG(ERR, "Failed to create unix socket: %s\n",
+			    strerror(errno));
+		return -1;
+	}
+	rte_strlcpy(un.sun_path, path, sizeof(un.sun_path));
+	if (connect(socket_fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
+		TESTPMD_LOG(ERR, "Failed to connect %s: %s\n", un.sun_path,
+			    strerror(errno));
+		close(socket_fd);
+		return -1;
+	}
+
+	/* Send the request message. */
+	do {
+		ret = sendmsg(socket_fd, &msgh, 0);
+	} while (ret < 0 && errno == EINTR);
+	if (ret < 0) {
+		TESTPMD_LOG(ERR, "Failed to send request to (%s): %s\n", path,
+			    strerror(errno));
+		close(socket_fd);
+		return -1;
+	}
+
+	msgh.msg_iov = &iov;
+	msgh.msg_iovlen = 1;
+	msgh.msg_control = control.buf;
+	msgh.msg_controllen = sizeof(control.buf);
+	do {
+		ret = recvmsg(socket_fd, &msgh, 0);
+	} while (ret < 0);
+	if (ret != sizeof(int) || (msgh.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
+		TESTPMD_LOG(ERR, "truncated msg");
+		close(socket_fd);
+		return -1;
+	}
+
+	/* Translate the FD. */
+	cmsg = CMSG_FIRSTHDR(&msgh);
+	if (cmsg == NULL || cmsg->cmsg_len != CMSG_LEN(sizeof(int)) ||
+	    cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) {
+		TESTPMD_LOG(ERR, "Fail to get FD using SCM_RIGHTS mechanism\n");
+		close(socket_fd);
+		unlink(un.sun_path);
+		return -1;
+	}
+	memcpy(&cmd_fd, CMSG_DATA(cmsg), sizeof(int));
+
+	TESTPMD_LOG(DEBUG, "Command FD (%d) and PD handle (%d) "
+		    "are successfully imported from remote process\n",
+		    cmd_fd, pd_handle);
+
+	/* Cleanup IPC channel. */
+	close(socket_fd);
+
+	/* Calculate the new length of devargs string. */
+	len += snprintf(NULL, 0, ",cmd_fd=%d,pd_handle=%d", cmd_fd, pd_handle);
+	/* Extend the devargs string. */
+	snprintf(extend, len, ",cmd_fd=%d,pd_handle=%d", cmd_fd, pd_handle);
+
+	TESTPMD_LOG(DEBUG, "Attach port with extra devargs %s\n", identifier);
+	return 0;
+}
+
+static bool
+is_delimiter_path_spaces(char *extend)
+{
+	while (*extend != '\0') {
+		if (*extend != ' ')
+			return true;
+		extend++;
+	}
+	return false;
+}
+
+int
+mlx5_test_attach_port_extend_devargs(char *identifier)
+{
+	char *extend = strchr(identifier, ' ');
+
+	if (extend != NULL && is_delimiter_path_spaces(extend) &&
+	    mlx5_test_extend_devargs(identifier, extend) < 0) {
+		TESTPMD_LOG(ERR, "Failed to extend devargs for port %s\n",
+			    identifier);
+		return -1;
+	}
+	return 0;
+}
+#endif
+
 /* *** SET HOST_SHAPER FOR A PORT *** */
 struct cmd_port_host_shaper_result {
 	cmdline_fixed_string_t mlx5;
diff --git a/drivers/net/mlx5/mlx5_testpmd.h b/drivers/net/mlx5/mlx5_testpmd.h
index 7a54658eb5..06976341a4 100644
--- a/drivers/net/mlx5/mlx5_testpmd.h
+++ b/drivers/net/mlx5/mlx5_testpmd.h
@@ -23,4 +23,20 @@
 void
 mlx5_test_avail_thresh_event_handler(uint16_t port_id, uint16_t rxq_id);
 
+/**
+ * Extend devargs list with "cmd_fd" and "pd_handle" coming from external
+ * process. It happens only in this format:
+ *  testpmd> port attach (identifier) mlx5_socket=<socket path>
+ * all "(identifier) mlx5_socket=<socket path>" is in the same string pointed
+ * by the input parameter 'identifier'.
+ *
+ * @param identifier
+ *   Identifier of port attach command line.
+ *
+ * @return
+ *   0 on success, -1 on failure.
+ */
+int
+mlx5_test_attach_port_extend_devargs(char *identifier);
+
 #endif
-- 
2.25.1