Re: vhost-net patches

["Michael S. Tsirkin" <mst@redhat.com>]

On Tue, Oct 27, 2009 at 09:36:18AM -0700, Shirley Ma wrote:
> Hello Michael,
> 
> On Tue, 2009-10-27 at 17:27 +0200, Michael S. Tsirkin wrote:
> > Possibly GFP_ATOMIC allocations in vring_add_indirect are failing?
> > Is there a chance you are tight on guest memory for some reason?
> > with vhost, virtio does currently consume a bit more memory than
> > with userspace backend.
> 
> I did see memory leak on host every time after exiting guest. I don't
> know where. Do you see it?
> 
> Anyway after I reboot host and restart guest with large memory
> allocation, I do see performance improves to 3xxxMb/s, and occasionally
> reaches 40xxMb/s. But "queue full" still exists, I can avoid the problem
> by increasing send queue size from qemu.

Here's another hack to try. It will break raw sockets,
but just as a test:

diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index a140dad..e830b30 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -106,22 +106,41 @@ static void handle_tx(struct vhost_net *net)
 		.msg_flags = MSG_DONTWAIT,
 	};
 	size_t len, total_len = 0;
-	int err;
+	int err, wmem;
 	size_t hdr_size;
 	struct socket *sock = rcu_dereference(vq->private_data);
-	if (!sock || !sock_writeable(sock->sk))
+	if (!sock)
+		return;
+
+	wmem = atomic_read(&sock->sk->sk_wmem_alloc);
+	if (wmem >= sock->sk->sk_sndbuf)
 		return;
 
 	use_mm(net->dev.mm);
 	mutex_lock(&vq->mutex);
-	tx_poll_stop(net);
+	vhost_no_notify(vq);
+	
+	if (wmem >= sock->sk->sk_sndbuf * 3 / 4) {
+		//tx_poll_start(net);
+	} else {
+		//tx_poll_stop(net);
+	}
 	hdr_size = vq->hdr_size;
 
 	for (;;) {
 		head = vhost_get_vq_desc(&net->dev, vq, vq->iov, &out, &in);
 		/* Nothing new?  Wait for eventfd to tell us they refilled. */
-		if (head == vq->num)
+		if (head == vq->num) {
+			wmem = atomic_read(&sock->sk->sk_wmem_alloc);
+			if (wmem >= sock->sk->sk_sndbuf * 3 / 4) {
+				set_bit(SOCK_ASYNC_NOSPACE, &sock->flags);
+				break;
+			}
+			if (vhost_notify(vq)) {
+				continue;
+			}
 			break;
+		}
 		if (in) {
 			vq_err(vq, "Unexpected descriptor format for TX: "
 			       "out %d, int %d\n", out, in);
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 30708c6..67bfc08 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -775,7 +775,7 @@ void vhost_no_notify(struct vhost_virtqueue *vq)
 
 int vhost_init(void)
 {
-	vhost_workqueue = create_workqueue("vhost");
+	vhost_workqueue = create_singlethread_workqueue("vhost");
 	if (!vhost_workqueue)
 		return -ENOMEM;
 	return 0;