[PATCH 1/2] kvm tools: Add optional callbacks for VQs

[PATCH 1/2] kvm tools: Add optional callbacks for VQs

[Sasha Levin]

This patch adds optional callbacks which get called when the VQ gets assigned
an eventfd for notifications, and when it gets assigned with a GSI.

This allows the device to pass the eventfds to 3rd parties which can use
them to notify and get notifications regarding the VQ.

Signed-off-by: Sasha Levin <levinsasha928@gmail.com>
---
 tools/kvm/include/kvm/virtio-trans.h |    2 ++
 tools/kvm/virtio/pci.c               |    6 ++++++
 2 files changed, 8 insertions(+), 0 deletions(-)

diff --git a/tools/kvm/include/kvm/virtio-trans.h b/tools/kvm/include/kvm/virtio-trans.h
index d9f4b95..e7c186e 100644
--- a/tools/kvm/include/kvm/virtio-trans.h
+++ b/tools/kvm/include/kvm/virtio-trans.h
@@ -20,6 +20,8 @@ struct virtio_ops {
 	int (*notify_vq)(struct kvm *kvm, void *dev, u32 vq);
 	int (*get_pfn_vq)(struct kvm *kvm, void *dev, u32 vq);
 	int (*get_size_vq)(struct kvm *kvm, void *dev, u32 vq);
+	void (*notify_vq_gsi)(struct kvm *kvm, void *dev, u32 vq, u32 gsi);
+	void (*notify_vq_eventfd)(struct kvm *kvm, void *dev, u32 vq, u32 efd);
 };
 
 struct virtio_trans_ops {
diff --git a/tools/kvm/virtio/pci.c b/tools/kvm/virtio/pci.c
index 1660f06..0737ae7 100644
--- a/tools/kvm/virtio/pci.c
+++ b/tools/kvm/virtio/pci.c
@@ -51,6 +51,9 @@ static int virtio_pci__init_ioeventfd(struct kvm *kvm, struct virtio_trans *vtra
 
 	ioeventfd__add_event(&ioevent);
 
+	if (vtrans->virtio_ops->notify_vq_eventfd)
+		vtrans->virtio_ops->notify_vq_eventfd(kvm, vpci->dev, vq, ioevent.fd);
+
 	return 0;
 }
 
@@ -152,6 +155,9 @@ static bool virtio_pci__specific_io_out(struct kvm *kvm, struct virtio_trans *vt
 
 			gsi = irq__add_msix_route(kvm, &vpci->msix_table[vec].msg);
 			vpci->gsis[vpci->queue_selector] = gsi;
+			if (vtrans->virtio_ops->notify_vq_gsi)
+				vtrans->virtio_ops->notify_vq_gsi(kvm, vpci->dev,
+							vpci->queue_selector, gsi);
 			break;
 		}
 		};
-- 
1.7.8.rc1

[PATCH 2/2] kvm tools: Add vhost-net support

[Sasha Levin]

This patch adds support to using the vhost-net device when using a tap backed
virtio-net device.

Activating vhost-net is done by appending a 'vhost=1' flag to the net device
configuration. For example:

	'kvm run -n mode=tap,vhost=1'

Cc: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Sasha Levin <levinsasha928@gmail.com>
---
 tools/kvm/builtin-run.c            |    2 +
 tools/kvm/include/kvm/virtio-net.h |    1 +
 tools/kvm/virtio/net.c             |  120 +++++++++++++++++++++++++++++++++++-
 3 files changed, 122 insertions(+), 1 deletions(-)

diff --git a/tools/kvm/builtin-run.c b/tools/kvm/builtin-run.c
index 13025db..3b00bf0 100644
--- a/tools/kvm/builtin-run.c
+++ b/tools/kvm/builtin-run.c
@@ -217,6 +217,8 @@ static int set_net_param(struct virtio_net_params *p, const char *param,
 		p->guest_ip = strdup(val);
 	} else if (strcmp(param, "host_ip") == 0) {
 		p->host_ip = strdup(val);
+	} else if (strcmp(param, "vhost") == 0) {
+		p->vhost = atoi(val);
 	}
 
 	return 0;
diff --git a/tools/kvm/include/kvm/virtio-net.h b/tools/kvm/include/kvm/virtio-net.h
index 58ae162..dade8cb 100644
--- a/tools/kvm/include/kvm/virtio-net.h
+++ b/tools/kvm/include/kvm/virtio-net.h
@@ -11,6 +11,7 @@ struct virtio_net_params {
 	char host_mac[6];
 	struct kvm *kvm;
 	int mode;
+	int vhost;
 };
 
 void virtio_net__init(const struct virtio_net_params *params);
diff --git a/tools/kvm/virtio/net.c b/tools/kvm/virtio/net.c
index cee2b5b..58ca4ed 100644
--- a/tools/kvm/virtio/net.c
+++ b/tools/kvm/virtio/net.c
@@ -10,6 +10,7 @@
 #include "kvm/guest_compat.h"
 #include "kvm/virtio-trans.h"
 
+#include <linux/vhost.h>
 #include <linux/virtio_net.h>
 #include <linux/if_tun.h>
 #include <linux/types.h>
@@ -25,6 +26,7 @@
 #include <sys/ioctl.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <sys/eventfd.h>
 
 #define VIRTIO_NET_QUEUE_SIZE		128
 #define VIRTIO_NET_NUM_QUEUES		2
@@ -57,6 +59,7 @@ struct net_dev {
 	pthread_mutex_t			io_tx_lock;
 	pthread_cond_t			io_tx_cond;
 
+	int				vhost_fd;
 	int				tap_fd;
 	char				tap_name[IFNAMSIZ];
 
@@ -323,9 +326,12 @@ static void set_guest_features(struct kvm *kvm, void *dev, u32 features)
 
 static int init_vq(struct kvm *kvm, void *dev, u32 vq, u32 pfn)
 {
+	struct vhost_vring_state state = { .index = vq };
+	struct vhost_vring_addr addr;
 	struct net_dev *ndev = dev;
 	struct virt_queue *queue;
 	void *p;
+	int r;
 
 	compat__remove_message(compat_id);
 
@@ -335,9 +341,82 @@ static int init_vq(struct kvm *kvm, void *dev, u32 vq, u32 pfn)
 
 	vring_init(&queue->vring, VIRTIO_NET_QUEUE_SIZE, p, VIRTIO_PCI_VRING_ALIGN);
 
+	if (ndev->vhost_fd == 0)
+		return 0;
+
+	state.num = queue->vring.num;
+	r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_NUM, &state);
+	if (r < 0)
+		die_perror("VHOST_SET_VRING_NUM failed");
+	state.num = 0;
+	r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_BASE, &state);
+	if (r < 0)
+		die_perror("VHOST_SET_VRING_BASE failed");
+
+	addr = (struct vhost_vring_addr) {
+		.index = vq,
+		.desc_user_addr = (u64)(unsigned long)queue->vring.desc,
+		.avail_user_addr = (u64)(unsigned long)queue->vring.avail,
+		.used_user_addr = (u64)(unsigned long)queue->vring.used,
+	};
+
+	r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_ADDR, &addr);
+	if (r < 0)
+		die_perror("VHOST_SET_VRING_ADDR failed");
+
 	return 0;
 }
 
+static void notify_vq_gsi(struct kvm *kvm, void *dev, u32 vq, u32 gsi)
+{
+	struct net_dev *ndev = dev;
+	struct kvm_irqfd irq;
+	struct vhost_vring_file file;
+	int r;
+
+	if (ndev->vhost_fd == 0)
+		return;
+
+	irq = (struct kvm_irqfd) {
+		.gsi	= gsi,
+		.fd	= eventfd(0, 0),
+	};
+	file = (struct vhost_vring_file) {
+		.index	= vq,
+		.fd	= irq.fd,
+	};
+
+	r = ioctl(kvm->vm_fd, KVM_IRQFD, &irq);
+	if (r < 0)
+		die_perror("KVM_IRQFD failed");
+
+	r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_CALL, &file);
+	if (r < 0)
+		die_perror("VHOST_SET_VRING_CALL failed");
+	file.fd = ndev->tap_fd;
+	r = ioctl(ndev->vhost_fd, VHOST_NET_SET_BACKEND, &file);
+	if (r != 0)
+		die("VHOST_NET_SET_BACKEND failed %d", errno);
+
+}
+
+static void notify_vq_eventfd(struct kvm *kvm, void *dev, u32 vq, u32 efd)
+{
+	struct net_dev *ndev = dev;
+	struct vhost_vring_file file = {
+		.index	= vq,
+		.fd	= efd,
+	};
+	int r;
+
+	if (ndev->vhost_fd == 0)
+		return;
+
+	r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_KICK, &file);
+	if (r < 0)
+		die_perror("VHOST_SET_VRING_KICK failed");
+}
+
 static int notify_vq(struct kvm *kvm, void *dev, u32 vq)
 {
 	struct net_dev *ndev = dev;
@@ -368,8 +447,44 @@ static struct virtio_ops net_dev_virtio_ops = (struct virtio_ops) {
 	.notify_vq		= notify_vq,
 	.get_pfn_vq		= get_pfn_vq,
 	.get_size_vq		= get_size_vq,
+	.notify_vq_gsi		= notify_vq_gsi,
+	.notify_vq_eventfd	= notify_vq_eventfd,
 };
 
+static void virtio_net__vhost_init(struct kvm *kvm, struct net_dev *ndev)
+{
+	u64 features = 0;
+	struct vhost_memory *mem;
+	int r;
+
+	ndev->vhost_fd = open("/dev/vhost-net", O_RDWR);
+	if (ndev->vhost_fd < 0)
+		die_perror("Failed openning vhost-net device");
+
+	mem = malloc(sizeof(*mem) + sizeof(struct vhost_memory_region));
+	if (mem == NULL)
+		die("Failed allocating memory for vhost memory map");
+
+	mem->nregions = 1;
+	mem->regions[0] = (struct vhost_memory_region) {
+		.guest_phys_addr	= 0,
+		.memory_size		= kvm->ram_size,
+		.userspace_addr		= (u64)kvm->ram_start,
+	};
+
+	r = ioctl(ndev->vhost_fd, VHOST_SET_OWNER);
+	if (r != 0)
+		die_perror("VHOST_SET_OWNER failed");
+
+	r = ioctl(ndev->vhost_fd, VHOST_SET_FEATURES, &features);
+	if (r != 0)
+		die_perror("VHOST_SET_FEATURES failed");
+	r = ioctl(ndev->vhost_fd, VHOST_SET_MEM_TABLE, mem);
+	if (r != 0)
+		die_perror("VHOST_SET_MEM_TABLE failed");
+	free(mem);
+}
+
 void virtio_net__init(const struct virtio_net_params *params)
 {
 	int i;
@@ -415,7 +530,10 @@ void virtio_net__init(const struct virtio_net_params *params)
 					VIRTIO_ID_NET, PCI_CLASS_NET);
 	ndev->vtrans.virtio_ops = &net_dev_virtio_ops;
 
-	virtio_net__io_thread_init(params->kvm, ndev);
+	if (params->vhost)
+		virtio_net__vhost_init(params->kvm, ndev);
+	else
+		virtio_net__io_thread_init(params->kvm, ndev);
 
 	if (compat_id != -1)
 		compat_id = compat__add_message("virtio-net device was not detected",
-- 
1.7.8.rc1

Re: [PATCH 2/2] kvm tools: Add vhost-net support

[Sasha Levin]

On Wed, 2011-11-16 at 14:24 +0200, Sasha Levin wrote:
> This patch adds support to using the vhost-net device when using a tap backed
> virtio-net device.
> 
> Activating vhost-net is done by appending a 'vhost=1' flag to the net device
> configuration. For example:
> 
> 	'kvm run -n mode=tap,vhost=1'
> 
> Cc: Michael S. Tsirkin <mst@redhat.com>
> Signed-off-by: Sasha Levin <levinsasha928@gmail.com>
> ---

I forgot to attach performance numbers to the changelog, so here they
are:

Short version
------------------

TCP Throughput: +29%
UDP Throughput: +10%
TCP Latency: -15%
UDP Latency: -12%


Long version
------------------

MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to
192.168.33.4 (192.168.33.4) port 0 AF_INET
Recv   Send    Send                          
Socket Socket  Message  Elapsed              
Size   Size    Size     Time     Throughput  
bytes  bytes   bytes    secs.    10^6bits/sec  

 87380  16384  16384    10.00    4895.04

MIGRATED UDP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to
192.168.33.4 (192.168.33.4) port 0 AF_INET
Socket  Message  Elapsed      Messages                
Size    Size     Time         Okay Errors   Throughput
bytes   bytes    secs            #      #   10^6bits/sec

229376   65507   10.00      125287      0    6565.60
229376           10.00      106910           5602.57

MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET
to 192.168.33.4 (192.168.33.4) port 0 AF_INET : first burst 0
Local /Remote
Socket Size   Request  Resp.   Elapsed  Trans.
Send   Recv   Size     Size    Time     Rate         
bytes  Bytes  bytes    bytes   secs.    per sec   

16384  87380  1        1       10.00    14811.55

MIGRATED UDP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET
to 192.168.33.4 (192.168.33.4) port 0 AF_INET : first burst 0
Local /Remote
Socket Size   Request  Resp.   Elapsed  Trans.
Send   Recv   Size     Size    Time     Rate         
bytes  Bytes  bytes    bytes   secs.    per sec   

229376 229376 1        1       10.00    16000.44   
229376 229376

After:

MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to
192.168.33.4 (192.168.33.4) port 0 AF_INET
Recv   Send    Send                          
Socket Socket  Message  Elapsed              
Size   Size    Size     Time     Throughput  
bytes  bytes   bytes    secs.    10^6bits/sec  

 87380  16384  16384    10.00    6340.74

MIGRATED UDP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to
192.168.33.4 (192.168.33.4) port 0 AF_INET
Socket  Message  Elapsed      Messages                
Size    Size     Time         Okay Errors   Throughput
bytes   bytes    secs            #      #   10^6bits/sec

229376   65507   10.00      131478      0    6890.09
229376           10.00      118136           6190.90

MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET
to 192.168.33.4 (192.168.33.4) port 0 AF_INET : first burst 0
Local /Remote
Socket Size   Request  Resp.   Elapsed  Trans.
Send   Recv   Size     Size    Time     Rate         
bytes  Bytes  bytes    bytes   secs.    per sec   

16384  87380  1        1       10.00    17126.10

MIGRATED UDP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET
to 192.168.33.4 (192.168.33.4) port 0 AF_INET : first burst 0
Local /Remote
Socket Size   Request  Resp.   Elapsed  Trans.
Send   Recv   Size     Size    Time     Rate         
bytes  Bytes  bytes    bytes   secs.    per sec   

229376 229376 1        1       10.00    17944.51

-- 

Sasha.