[dpdk-dev] [PATCH 6/9] vfio_user: add client APIs of device attach/detach

[Chenbo Xia <chenbo.xia@intel.com>]

This patch implements two APIs, rte_vfio_user_attach_dev() and
rte_vfio_user_detach_dev() for vfio-user client to connect to
or disconnect from a vfio-user device on server side.

Signed-off-by: Chenbo Xia <chenbo.xia@intel.com>
Signed-off-by: Xiuchun Lu <xiuchun.lu@intel.com>
---
 lib/librte_vfio_user/meson.build        |   3 +-
 lib/librte_vfio_user/rte_vfio_user.h    |  30 +++
 lib/librte_vfio_user/version.map        |   2 +
 lib/librte_vfio_user/vfio_user_client.c | 279 ++++++++++++++++++++++++
 lib/librte_vfio_user/vfio_user_client.h |  25 +++
 5 files changed, 338 insertions(+), 1 deletion(-)
 create mode 100644 lib/librte_vfio_user/vfio_user_client.c
 create mode 100644 lib/librte_vfio_user/vfio_user_client.h

diff --git a/lib/librte_vfio_user/meson.build b/lib/librte_vfio_user/meson.build
index b7363f61c6..5761f0edd1 100644
--- a/lib/librte_vfio_user/meson.build
+++ b/lib/librte_vfio_user/meson.build
@@ -6,5 +6,6 @@ if not is_linux
 	reason = 'only supported on Linux'
 endif
 
-sources = files('vfio_user_base.c', 'vfio_user_server.c')
+sources = files('vfio_user_base.c', 'vfio_user_server.c',
+	'vfio_user_client.c')
 headers = files('rte_vfio_user.h')
diff --git a/lib/librte_vfio_user/rte_vfio_user.h b/lib/librte_vfio_user/rte_vfio_user.h
index 6c12b0b6ed..b09d83e224 100644
--- a/lib/librte_vfio_user/rte_vfio_user.h
+++ b/lib/librte_vfio_user/rte_vfio_user.h
@@ -225,4 +225,34 @@ __rte_experimental
 int rte_vfio_user_set_irq_info(const char *sock_addr,
 	struct rte_vfio_user_irq_info *irq);
 
+/**
+ *  Below APIs are for vfio-user client (device consumer) to use:
+ *	*rte_vfio_user_attach_dev
+ *	*rte_vfio_user_detach_dev
+ */
+
+/**
+ * Attach to a vfio-user device.
+ *
+ * @param sock_addr
+ *   Unix domain socket address
+ * @return
+ *   - >=0: Success, device attached. Returned value is the device ID.
+ *   - <0: Failure on device attach
+ */
+__rte_experimental
+int rte_vfio_user_attach_dev(const char *sock_addr);
+
+/**
+ * Detach from a vfio-user device.
+ *
+ * @param dev_id
+ *   Device ID of the vfio-user device
+ * @return
+ *   - 0: Success, device detached
+ *   - <0: Failure on device detach
+ */
+__rte_experimental
+int rte_vfio_user_detach_dev(int dev_id);
+
 #endif
diff --git a/lib/librte_vfio_user/version.map b/lib/librte_vfio_user/version.map
index 621a51a9fc..a0cda2b49c 100644
--- a/lib/librte_vfio_user/version.map
+++ b/lib/librte_vfio_user/version.map
@@ -10,6 +10,8 @@ EXPERIMENTAL {
 	rte_vfio_user_set_dev_info;
 	rte_vfio_user_set_reg_info;
 	rte_vfio_user_set_irq_info;
+	rte_vfio_user_attach_dev;
+	rte_vfio_user_detach_dev;
 
 	local: *;
 };
diff --git a/lib/librte_vfio_user/vfio_user_client.c b/lib/librte_vfio_user/vfio_user_client.c
new file mode 100644
index 0000000000..85b2e8cb46
--- /dev/null
+++ b/lib/librte_vfio_user/vfio_user_client.c
@@ -0,0 +1,279 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include <unistd.h>
+#include <string.h>
+#include <pthread.h>
+#include <fcntl.h>
+#include <sys/un.h>
+#include <sys/socket.h>
+
+#include "vfio_user_client.h"
+#include "rte_vfio_user.h"
+
+#define REPLY_USEC 1000
+#define RECV_MAX_TRY 50
+
+static struct vfio_user_client_devs vfio_client_devs = {
+	.cl_num = 0,
+	.mutex = PTHREAD_MUTEX_INITIALIZER,
+};
+
+/* Check if the sock_addr exists. If not, alloc and return index */
+static int vfio_user_client_allocate(const char *sock_addr)
+{
+	uint32_t i, count = 0;
+	int index = -1;
+
+	if (sock_addr == NULL)
+		return -1;
+
+	if (vfio_client_devs.cl_num == 0)
+		return 0;
+
+	for (i = 0; i < MAX_VFIO_USER_CLIENT; i++) {
+		struct vfio_user_client *cl = vfio_client_devs.cl[i];
+
+		if (!cl) {
+			if (index == -1)
+				index = i;
+			continue;
+		}
+
+		if (!strcmp(cl->sock.sock_addr, sock_addr))
+			return -1;
+
+		count++;
+		if (count == vfio_client_devs.cl_num)
+			break;
+	}
+
+	return index;
+}
+
+static struct vfio_user_client *
+vfio_user_client_create_dev(const char *sock_addr)
+{
+	struct vfio_user_client *cl;
+	struct vfio_user_socket *sock;
+	int fd, idx;
+	struct sockaddr_un un;
+
+	pthread_mutex_lock(&vfio_client_devs.mutex);
+	if (vfio_client_devs.cl_num == MAX_VFIO_USER_CLIENT) {
+		VFIO_USER_LOG(ERR, "Failed to create client:"
+			" client num reaches max\n");
+		goto err;
+	}
+
+	idx = vfio_user_client_allocate(sock_addr);
+	if (idx < 0) {
+		VFIO_USER_LOG(ERR, "Failed to create client:"
+			"socket addr exists\n");
+		goto err;
+	}
+
+	cl = malloc(sizeof(*cl));
+	if (!cl) {
+		VFIO_USER_LOG(ERR, "Failed to alloc client\n");
+		goto err;
+	}
+
+	sock = &cl->sock;
+	sock->sock_addr = strdup(sock_addr);
+	if (!sock->sock_addr) {
+		VFIO_USER_LOG(ERR, "Failed to copy sock_addr\n");
+		goto err_dup;
+	}
+
+	fd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (fd < 0) {
+		VFIO_USER_LOG(ERR, "Client failed to create socket: %s\n",
+			strerror(errno));
+		goto err_sock;
+	}
+
+	if (fcntl(fd, F_SETFL, O_NONBLOCK)) {
+		VFIO_USER_LOG(ERR, "Failed to set nonblocking mode for client "
+			"socket, fd: %d (%s)\n", fd, strerror(errno));
+		goto err_ctl;
+	}
+
+	memset(&un, 0, sizeof(un));
+	un.sun_family = AF_UNIX;
+	strncpy(un.sun_path, sock->sock_addr, sizeof(un.sun_path));
+	un.sun_path[sizeof(un.sun_path) - 1] = '\0';
+
+	if (connect(fd, &un, sizeof(un)) < 0) {
+		VFIO_USER_LOG(ERR, "Client connect error, %s\n",
+			strerror(errno));
+		goto err_ctl;
+	}
+
+	sock->sock_fd = fd;
+	sock->dev_id = idx;
+
+	vfio_client_devs.cl[idx] = cl;
+	vfio_client_devs.cl_num++;
+
+	pthread_mutex_unlock(&vfio_client_devs.mutex);
+
+	return cl;
+
+err_ctl:
+	close(fd);
+err_sock:
+	free(sock->sock_addr);
+err_dup:
+	free(sock);
+err:
+	pthread_mutex_unlock(&vfio_client_devs.mutex);
+	return NULL;
+}
+
+static int vfio_user_client_destroy_dev(int dev_id)
+{
+	struct vfio_user_client *cl;
+	struct vfio_user_socket *sock;
+	int ret = 0;
+
+	pthread_mutex_lock(&vfio_client_devs.mutex);
+	if (vfio_client_devs.cl_num == 0) {
+		VFIO_USER_LOG(ERR, "Failed to destroy client:"
+			" no client exists\n");
+		ret = -EINVAL;
+		goto err;
+	}
+
+	cl = vfio_client_devs.cl[dev_id];
+	if (!cl) {
+		VFIO_USER_LOG(ERR, "Failed to destroy client:"
+			" wrong device ID(%d)\n", dev_id);
+		ret = -EINVAL;
+		goto err;
+	}
+
+	sock = &cl->sock;
+	free(sock->sock_addr);
+	close(sock->sock_fd);
+
+	free(cl);
+	vfio_client_devs.cl[dev_id] = NULL;
+	vfio_client_devs.cl_num--;
+
+err:
+	pthread_mutex_unlock(&vfio_client_devs.mutex);
+	return ret;
+}
+
+static inline void vfio_user_client_fill_hdr(VFIO_USER_MSG *msg, uint16_t cmd,
+	uint32_t sz)
+{
+	static uint16_t	msg_id;
+
+	msg->msg_id = msg_id++;
+	msg->cmd = cmd;
+	msg->size = sz;
+	msg->flags = VFIO_USER_TYPE_CMD;
+	msg->err = 0;
+}
+
+static int vfio_user_client_send_recv(int sock_fd, VFIO_USER_MSG *msg)
+{
+	uint16_t cmd = msg->cmd;
+	uint16_t id = msg->msg_id;
+	uint8_t try_recv = 0;
+	int ret;
+
+	ret = vfio_user_send_msg(sock_fd, msg);
+	if (ret < 0) {
+		VFIO_USER_LOG(ERR, "Send error for %s\n",
+			vfio_user_msg_str[cmd]);
+		return -1;
+	}
+
+	VFIO_USER_LOG(INFO, "Send request %s\n", vfio_user_msg_str[cmd]);
+
+	memset(msg, 0, sizeof(*msg));
+
+	while (try_recv < RECV_MAX_TRY) {
+		ret = vfio_user_recv_msg(sock_fd, msg);
+		if (!ret) {
+			VFIO_USER_LOG(ERR, "Peer closed\n");
+			return -1;
+		} else if (ret > 0) {
+			if (id != msg->msg_id)
+				continue;
+			else
+				break;
+		}
+		usleep(REPLY_USEC);
+		try_recv++;
+	}
+
+	if (cmd != msg->cmd) {
+		VFIO_USER_LOG(ERR, "Request and reply mismatch\n");
+		ret = -1;
+	} else
+		ret = 0;
+
+	VFIO_USER_LOG(INFO, "Recv reply %s\n", vfio_user_msg_str[cmd]);
+
+	return ret;
+}
+
+int rte_vfio_user_attach_dev(const char *sock_addr)
+{
+	struct vfio_user_client *dev;
+	VFIO_USER_MSG msg;
+	uint32_t sz = VFIO_USER_MSG_HDR_SIZE + sizeof(struct vfio_user_version)
+		- VFIO_USER_MAX_VERSION_DATA;
+	struct vfio_user_version *ver = &msg.payload.ver;
+	int ret;
+
+	if (!sock_addr)
+		return -EINVAL;
+
+	dev = vfio_user_client_create_dev(sock_addr);
+	if (!dev) {
+		VFIO_USER_LOG(ERR, "Failed to attach the device "
+			"with sock_addr %s\n", sock_addr);
+		return -1;
+	}
+
+	memset(&msg, 0, sizeof(VFIO_USER_MSG));
+	vfio_user_client_fill_hdr(&msg, VFIO_USER_VERSION, sz);
+	ver->major = VFIO_USER_VERSION_MAJOR;
+	ver->minor = VFIO_USER_VERSION_MINOR;
+
+	ret = vfio_user_client_send_recv(dev->sock.sock_fd, &msg);
+	if (ret)
+		return ret;
+
+	if (msg.flags & VFIO_USER_ERROR) {
+		VFIO_USER_LOG(ERR, "Failed to negotiate version: %s\n",
+				msg.err ? strerror(msg.err) : "Unknown error");
+		return msg.err ? -(int)msg.err : -1;
+	}
+
+	if (vfio_user_check_msg_fdnum(&msg, 0) != 0)
+		return -1;
+
+	return dev->sock.dev_id;
+}
+
+int rte_vfio_user_detach_dev(int dev_id)
+{
+	int ret;
+
+	if (dev_id < 0)
+		return -EINVAL;
+
+	ret = vfio_user_client_destroy_dev(dev_id);
+	if (ret)
+		VFIO_USER_LOG(ERR, "Failed to detach the device (ID:%d)\n",
+			dev_id);
+
+	return ret;
+}
diff --git a/lib/librte_vfio_user/vfio_user_client.h b/lib/librte_vfio_user/vfio_user_client.h
new file mode 100644
index 0000000000..d489e62037
--- /dev/null
+++ b/lib/librte_vfio_user/vfio_user_client.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#ifndef _VFIO_USER_CLIENT_H
+#define _VFIO_USER_CLIENT_H
+
+#include <stdint.h>
+
+#include "vfio_user_base.h"
+
+#define MAX_VFIO_USER_CLIENT 1024
+
+struct vfio_user_client {
+	struct vfio_user_socket sock;
+	uint8_t rsvd[16];	/* Reserved for future use */
+};
+
+struct vfio_user_client_devs {
+	struct vfio_user_client *cl[MAX_VFIO_USER_CLIENT];
+	uint32_t cl_num;
+	pthread_mutex_t mutex;
+};
+
+#endif
-- 
2.17.1