[PATCH v3 04/22] vhost: make notify ops per vhost driver

[Yuanhan Liu <yuanhan.liu@linux.intel.com>]

Assume there is an application both support vhost-user net and
vhost-user scsi, the callback should be different. Making notify
ops per vhost driver allow application define different set of
callbacks for different driver.

Signed-off-by: Yuanhan Liu <yuanhan.liu@linux.intel.com>
---

v2: - check the return value of callback_register and callback_get
    - update release note
---
 doc/guides/prog_guide/vhost_lib.rst    |  2 +-
 doc/guides/rel_notes/release_17_05.rst |  3 +++
 drivers/net/vhost/rte_eth_vhost.c      | 20 +++++++++++---------
 examples/tep_termination/main.c        |  7 ++++++-
 examples/vhost/main.c                  |  9 +++++++--
 lib/librte_vhost/rte_virtio_net.h      |  3 ++-
 lib/librte_vhost/socket.c              | 32 ++++++++++++++++++++++++++++++++
 lib/librte_vhost/vhost.c               | 16 +---------------
 lib/librte_vhost/vhost.h               |  5 ++++-
 lib/librte_vhost/vhost_user.c          | 22 ++++++++++++++++------
 10 files changed, 83 insertions(+), 36 deletions(-)

diff --git a/doc/guides/prog_guide/vhost_lib.rst b/doc/guides/prog_guide/vhost_lib.rst
index 6a4d206..40f3b3b 100644
--- a/doc/guides/prog_guide/vhost_lib.rst
+++ b/doc/guides/prog_guide/vhost_lib.rst
@@ -122,7 +122,7 @@ The following is an overview of some key Vhost API functions:
   starts an infinite loop, therefore it should be called in a dedicated
   thread.
 
-* ``rte_vhost_driver_callback_register(virtio_net_device_ops)``
+* ``rte_vhost_driver_callback_register(path, virtio_net_device_ops)``
 
   This function registers a set of callbacks, to let DPDK applications take
   the appropriate action when some events happen. The following events are
diff --git a/doc/guides/rel_notes/release_17_05.rst b/doc/guides/rel_notes/release_17_05.rst
index 4e405b1..dfa636d 100644
--- a/doc/guides/rel_notes/release_17_05.rst
+++ b/doc/guides/rel_notes/release_17_05.rst
@@ -135,6 +135,9 @@ API Changes
      * ``rte_eth_vhost_feature_enable``
      * ``rte_eth_vhost_feature_get``
 
+   * The vhost API ``rte_vhost_driver_callback_register(ops)`` takes one
+     more argument: ``rte_vhost_driver_callback_register(path, ops)``.
+
 
 ABI Changes
 -----------
diff --git a/drivers/net/vhost/rte_eth_vhost.c b/drivers/net/vhost/rte_eth_vhost.c
index 83063c2..f6ad616 100644
--- a/drivers/net/vhost/rte_eth_vhost.c
+++ b/drivers/net/vhost/rte_eth_vhost.c
@@ -669,6 +669,12 @@ struct vhost_xstats_name_off {
 	return 0;
 }
 
+static struct virtio_net_device_ops vhost_ops = {
+	.new_device          = new_device,
+	.destroy_device      = destroy_device,
+	.vring_state_changed = vring_state_changed,
+};
+
 int
 rte_eth_vhost_get_queue_event(uint8_t port_id,
 		struct rte_eth_vhost_queue_event *event)
@@ -738,15 +744,6 @@ struct vhost_xstats_name_off {
 static void *
 vhost_driver_session(void *param __rte_unused)
 {
-	static struct virtio_net_device_ops vhost_ops;
-
-	/* set vhost arguments */
-	vhost_ops.new_device = new_device;
-	vhost_ops.destroy_device = destroy_device;
-	vhost_ops.vring_state_changed = vring_state_changed;
-	if (rte_vhost_driver_callback_register(&vhost_ops) < 0)
-		RTE_LOG(ERR, PMD, "Can't register callbacks\n");
-
 	/* start event handling */
 	rte_vhost_driver_session_start();
 
@@ -1079,6 +1076,11 @@ struct vhost_xstats_name_off {
 	if (rte_vhost_driver_register(iface_name, flags))
 		goto error;
 
+	if (rte_vhost_driver_callback_register(iface_name, &vhost_ops) < 0) {
+		RTE_LOG(ERR, PMD, "Can't register callbacks\n");
+		goto error;
+	}
+
 	/* We need only one message handling thread */
 	if (rte_atomic16_add_return(&nb_started_ports, 1) == 1) {
 		if (vhost_driver_session_start())
diff --git a/examples/tep_termination/main.c b/examples/tep_termination/main.c
index 8097dcd..18b977e 100644
--- a/examples/tep_termination/main.c
+++ b/examples/tep_termination/main.c
@@ -1256,7 +1256,12 @@ static inline void __attribute__((always_inline))
 	rte_vhost_driver_disable_features(dev_basename,
 		1ULL << VIRTIO_NET_F_MRG_RXBUF);
 
-	rte_vhost_driver_callback_register(&virtio_net_device_ops);
+	ret = rte_vhost_driver_callback_register(dev_basename,
+		&virtio_net_device_ops);
+	if (ret != 0) {
+		rte_exit(EXIT_FAILURE,
+			"failed to register vhost driver callbacks.\n");
+	}
 
 	rte_vhost_driver_session_start();
 
diff --git a/examples/vhost/main.c b/examples/vhost/main.c
index 972a6a8..72a9d69 100644
--- a/examples/vhost/main.c
+++ b/examples/vhost/main.c
@@ -1538,9 +1538,14 @@ static inline void __attribute__((always_inline))
 			rte_vhost_driver_enable_features(file,
 				1ULL << VIRTIO_NET_F_CTRL_RX);
 		}
-	}
 
-	rte_vhost_driver_callback_register(&virtio_net_device_ops);
+		ret = rte_vhost_driver_callback_register(file,
+			&virtio_net_device_ops);
+		if (ret != 0) {
+			rte_exit(EXIT_FAILURE,
+				"failed to register vhost driver callbacks.\n");
+		}
+	}
 
 	rte_vhost_driver_session_start();
 	return 0;
diff --git a/lib/librte_vhost/rte_virtio_net.h b/lib/librte_vhost/rte_virtio_net.h
index 5dadd3d..67bd125 100644
--- a/lib/librte_vhost/rte_virtio_net.h
+++ b/lib/librte_vhost/rte_virtio_net.h
@@ -133,7 +133,8 @@ struct virtio_net_device_ops {
 uint64_t rte_vhost_driver_get_features(const char *path);
 
 /* Register callbacks. */
-int rte_vhost_driver_callback_register(struct virtio_net_device_ops const * const);
+int rte_vhost_driver_callback_register(const char *path,
+	struct virtio_net_device_ops const * const ops);
 /* Start vhost driver session blocking loop. */
 int rte_vhost_driver_session_start(void);
 
diff --git a/lib/librte_vhost/socket.c b/lib/librte_vhost/socket.c
index 87801c0..ed9c5b4 100644
--- a/lib/librte_vhost/socket.c
+++ b/lib/librte_vhost/socket.c
@@ -77,6 +77,8 @@ struct vhost_user_socket {
 	 */
 	uint64_t supported_features;
 	uint64_t features;
+
+	struct virtio_net_device_ops const *notify_ops;
 };
 
 struct vhost_user_connection {
@@ -741,6 +743,36 @@ struct vhost_user_reconnect_list {
 	return -1;
 }
 
+/*
+ * Register ops so that we can add/remove device to data core.
+ */
+int
+rte_vhost_driver_callback_register(const char *path,
+	struct virtio_net_device_ops const * const ops)
+{
+	struct vhost_user_socket *vsocket;
+
+	pthread_mutex_lock(&vhost_user.mutex);
+	vsocket = find_vhost_user_socket(path);
+	if (vsocket)
+		vsocket->notify_ops = ops;
+	pthread_mutex_unlock(&vhost_user.mutex);
+
+	return vsocket ? 0 : -1;
+}
+
+struct virtio_net_device_ops const *
+vhost_driver_callback_get(const char *path)
+{
+	struct vhost_user_socket *vsocket;
+
+	pthread_mutex_lock(&vhost_user.mutex);
+	vsocket = find_vhost_user_socket(path);
+	pthread_mutex_unlock(&vhost_user.mutex);
+
+	return vsocket ? vsocket->notify_ops : NULL;
+}
+
 int
 rte_vhost_driver_session_start(void)
 {
diff --git a/lib/librte_vhost/vhost.c b/lib/librte_vhost/vhost.c
index 7b40a92..7d7bb3c 100644
--- a/lib/librte_vhost/vhost.c
+++ b/lib/librte_vhost/vhost.c
@@ -51,9 +51,6 @@
 
 struct virtio_net *vhost_devices[MAX_VHOST_DEVICE];
 
-/* device ops to add/remove device to/from data core. */
-struct virtio_net_device_ops const *notify_ops;
-
 struct virtio_net *
 get_device(int vid)
 {
@@ -253,7 +250,7 @@ struct virtio_net *
 
 	if (dev->flags & VIRTIO_DEV_RUNNING) {
 		dev->flags &= ~VIRTIO_DEV_RUNNING;
-		notify_ops->destroy_device(vid);
+		dev->notify_ops->destroy_device(vid);
 	}
 
 	cleanup_device(dev, 1);
@@ -396,14 +393,3 @@ struct virtio_net *
 	dev->virtqueue[queue_id]->used->flags = VRING_USED_F_NO_NOTIFY;
 	return 0;
 }
-
-/*
- * Register ops so that we can add/remove device to data core.
- */
-int
-rte_vhost_driver_callback_register(struct virtio_net_device_ops const * const ops)
-{
-	notify_ops = ops;
-
-	return 0;
-}
diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h
index 692691b..6186216 100644
--- a/lib/librte_vhost/vhost.h
+++ b/lib/librte_vhost/vhost.h
@@ -185,6 +185,8 @@ struct virtio_net {
 	struct ether_addr	mac;
 	uint16_t		mtu;
 
+	struct virtio_net_device_ops const *notify_ops;
+
 	uint32_t		nr_guest_pages;
 	uint32_t		max_guest_pages;
 	struct guest_page       *guest_pages;
@@ -288,7 +290,6 @@ static inline phys_addr_t __attribute__((always_inline))
 	return 0;
 }
 
-struct virtio_net_device_ops const *notify_ops;
 struct virtio_net *get_device(int vid);
 
 int vhost_new_device(void);
@@ -301,6 +302,8 @@ static inline phys_addr_t __attribute__((always_inline))
 void vhost_set_ifname(int, const char *if_name, unsigned int if_len);
 void vhost_enable_dequeue_zero_copy(int vid);
 
+struct virtio_net_device_ops const *vhost_driver_callback_get(const char *path);
+
 /*
  * Backend-specific cleanup.
  *
diff --git a/lib/librte_vhost/vhost_user.c b/lib/librte_vhost/vhost_user.c
index d630098..0cadd79 100644
--- a/lib/librte_vhost/vhost_user.c
+++ b/lib/librte_vhost/vhost_user.c
@@ -135,7 +135,7 @@
 {
 	if (dev->flags & VIRTIO_DEV_RUNNING) {
 		dev->flags &= ~VIRTIO_DEV_RUNNING;
-		notify_ops->destroy_device(dev->vid);
+		dev->notify_ops->destroy_device(dev->vid);
 	}
 
 	cleanup_device(dev, 0);
@@ -503,7 +503,7 @@
 	/* Remove from the data plane. */
 	if (dev->flags & VIRTIO_DEV_RUNNING) {
 		dev->flags &= ~VIRTIO_DEV_RUNNING;
-		notify_ops->destroy_device(dev->vid);
+		dev->notify_ops->destroy_device(dev->vid);
 	}
 
 	if (dev->mem) {
@@ -687,7 +687,7 @@
 						"dequeue zero copy is enabled\n");
 			}
 
-			if (notify_ops->new_device(dev->vid) == 0)
+			if (dev->notify_ops->new_device(dev->vid) == 0)
 				dev->flags |= VIRTIO_DEV_RUNNING;
 		}
 	}
@@ -721,7 +721,7 @@
 	/* We have to stop the queue (virtio) if it is running. */
 	if (dev->flags & VIRTIO_DEV_RUNNING) {
 		dev->flags &= ~VIRTIO_DEV_RUNNING;
-		notify_ops->destroy_device(dev->vid);
+		dev->notify_ops->destroy_device(dev->vid);
 	}
 
 	dev->flags &= ~VIRTIO_DEV_READY;
@@ -763,8 +763,8 @@
 		"set queue enable: %d to qp idx: %d\n",
 		enable, state->index);
 
-	if (notify_ops->vring_state_changed)
-		notify_ops->vring_state_changed(dev->vid, state->index, enable);
+	if (dev->notify_ops->vring_state_changed)
+		dev->notify_ops->vring_state_changed(dev->vid, state->index, enable);
 
 	dev->virtqueue[state->index]->enabled = enable;
 
@@ -978,6 +978,16 @@
 	if (dev == NULL)
 		return -1;
 
+	if (!dev->notify_ops) {
+		dev->notify_ops = vhost_driver_callback_get(dev->ifname);
+		if (!dev->notify_ops) {
+			RTE_LOG(ERR, VHOST_CONFIG,
+				"failed to get callback ops for driver %s\n",
+				dev->ifname);
+			return -1;
+		}
+	}
+
 	ret = read_vhost_message(fd, &msg);
 	if (ret <= 0 || msg.request >= VHOST_USER_MAX) {
 		if (ret < 0)
-- 
1.9.0