[PATCH RFC 1/2] virtio_net: implement VIRTIO_CONFIG_S_NEEDS_RESET

[PATCH RFC 1/2] virtio_net: implement VIRTIO_CONFIG_S_NEEDS_RESET

[Willem de Bruijn]

From: Willem de Bruijn <willemb@google.com>

Implement the reset communication request defined in the VIRTIO 1.0
specification and introduces in Linux in commit c00bbcf862896 ("virtio:
add VIRTIO_CONFIG_S_NEEDS_RESET device status bit").

Since that patch, the virtio-net driver has added a virtnet_reset
function that implements the requested behavior through calls to the
power management freeze and restore functions.

That function has recently been reverted when its sole caller was
updated. Bring it back and listen for the request from the host on
the config channel.

Implement the feature analogous to other config requests. In
particular, acknowledge the request in the same manner as the
NET_S_ANNOUNCE link announce request, by responding with a new
VIRTIO_NET_CTRL_${TYPE} command. On reception, the host must check
the config status register for success or failure.

The existing freeze handler verifies that no config changes are
running concurrently. Elide this check for reset. The request is
always handled from the config workqueue. No other config requests
can be active or scheduled concurrently on vi->config.

Signed-off-by: Willem de Bruijn <willemb@google.com>
---
 drivers/net/virtio_net.c        | 69 +++++++++++++++++++++++++++++++++++------
 include/uapi/linux/virtio_net.h |  4 +++
 2 files changed, 64 insertions(+), 9 deletions(-)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 52ae78ca3d38..5e349226f7c1 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -1458,12 +1458,11 @@ static void virtnet_netpoll(struct net_device *dev)
 }
 #endif
 
-static void virtnet_ack_link_announce(struct virtnet_info *vi)
+static void virtnet_ack(struct virtnet_info *vi, u8 class, u8 cmd)
 {
 	rtnl_lock();
-	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
-				  VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
-		dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
+	if (!virtnet_send_command(vi, class, cmd, NULL))
+		dev_warn(&vi->dev->dev, "Failed to ack %u.%u\n", class, cmd);
 	rtnl_unlock();
 }
 
@@ -1857,13 +1856,16 @@ static const struct ethtool_ops virtnet_ethtool_ops = {
 	.set_link_ksettings = virtnet_set_link_ksettings,
 };
 
-static void virtnet_freeze_down(struct virtio_device *vdev)
+static void virtnet_freeze_down(struct virtio_device *vdev, bool in_reset)
 {
 	struct virtnet_info *vi = vdev->priv;
 	int i;
 
-	/* Make sure no work handler is accessing the device */
-	flush_work(&vi->config_work);
+	/* Make sure no work handler is accessing the device,
+	 * unless this call is made from the reset work handler itself.
+	 */
+	if (!in_reset)
+		flush_work(&vi->config_work);
 
 	netif_device_detach(vi->dev);
 	netif_tx_disable(vi->dev);
@@ -1878,6 +1880,7 @@ static void virtnet_freeze_down(struct virtio_device *vdev)
 }
 
 static int init_vqs(struct virtnet_info *vi);
+static void remove_vq_common(struct virtnet_info *vi);
 
 static int virtnet_restore_up(struct virtio_device *vdev)
 {
@@ -1906,6 +1909,45 @@ static int virtnet_restore_up(struct virtio_device *vdev)
 	return err;
 }
 
+static int virtnet_reset(struct virtnet_info *vi)
+{
+	struct virtio_device *dev = vi->vdev;
+	bool failed;
+	int ret;
+
+	virtio_config_disable(dev);
+	failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
+	virtnet_freeze_down(dev, true);
+	remove_vq_common(vi);
+
+	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
+	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
+
+	/* Restore the failed status (see virtio_device_restore). */
+	if (failed)
+		virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
+
+	ret = virtio_finalize_features(dev);
+	if (ret)
+		goto err;
+
+	ret = virtnet_restore_up(dev);
+	if (ret)
+		goto err;
+
+	ret = virtnet_set_queues(vi, vi->curr_queue_pairs);
+	if (ret)
+		goto err;
+
+	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
+	virtio_config_enable(dev);
+	return 0;
+
+err:
+	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
+	return ret;
+}
+
 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
 {
 	struct scatterlist sg;
@@ -2085,7 +2127,16 @@ static void virtnet_config_changed_work(struct work_struct *work)
 
 	if (v & VIRTIO_NET_S_ANNOUNCE) {
 		netdev_notify_peers(vi->dev);
-		virtnet_ack_link_announce(vi);
+		virtnet_ack(vi, VIRTIO_NET_CTRL_ANNOUNCE,
+			    VIRTIO_NET_CTRL_ANNOUNCE_ACK);
+	}
+
+	if (vi->vdev->config->get_status(vi->vdev) &
+	    VIRTIO_CONFIG_S_NEEDS_RESET) {
+		virtnet_reset(vi);
+		virtnet_ack(vi, VIRTIO_NET_CTRL_RESET,
+			    VIRTIO_NET_CTRL_RESET_ACK);
+
 	}
 
 	/* Ignore unknown (future) status bits */
@@ -2708,7 +2759,7 @@ static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
 	struct virtnet_info *vi = vdev->priv;
 
 	virtnet_cpu_notif_remove(vi);
-	virtnet_freeze_down(vdev);
+	virtnet_freeze_down(vdev, false);
 	remove_vq_common(vi);
 
 	return 0;
diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h
index fc353b518288..188fdc528f13 100644
--- a/include/uapi/linux/virtio_net.h
+++ b/include/uapi/linux/virtio_net.h
@@ -245,4 +245,8 @@ struct virtio_net_ctrl_mq {
 #define VIRTIO_NET_CTRL_GUEST_OFFLOADS   5
 #define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET        0
 
+/* Signal that the driver received and executed the reset command. */
+#define VIRTIO_NET_CTRL_RESET			6
+#define VIRTIO_NET_CTRL_RESET_ACK		0
+
 #endif /* _UAPI_LINUX_VIRTIO_NET_H */
-- 
2.14.1.342.g6490525c54-goog

[PATCH RFC 2/2] virtio_net: enable probing for NEEDS_RESET support

[Willem de Bruijn]

From: Willem de Bruijn <willemb@google.com>

Implement a mechanism to signal that a virtio device implements the
VIRTIO_CONFIG_S_NEEDS_RESET command.

Testing for VIRTIO_CONFIG_S_NEEDS_RESET support by issuing the request
and verifying the reset state would require an expensive state change.

To avoid that, add a status bit to the feature register used during
feature negotiation between host and guest.

Set the bit for virtio-net, which supports the feature.

Signed-off-by: Willem de Bruijn <willemb@google.com>
---
 drivers/net/virtio_net.c           | 2 ++
 include/uapi/linux/virtio_config.h | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 5e349226f7c1..15483a982106 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -2801,12 +2801,14 @@ static struct virtio_device_id id_table[] = {
 
 static unsigned int features[] = {
 	VIRTNET_FEATURES,
+	VIRTIO_F_CAN_RESET,
 };
 
 static unsigned int features_legacy[] = {
 	VIRTNET_FEATURES,
 	VIRTIO_NET_F_GSO,
 	VIRTIO_F_ANY_LAYOUT,
+	VIRTIO_F_CAN_RESET,
 };
 
 static struct virtio_driver virtio_net_driver = {
diff --git a/include/uapi/linux/virtio_config.h b/include/uapi/linux/virtio_config.h
index 308e2096291f..726be44a04d3 100644
--- a/include/uapi/linux/virtio_config.h
+++ b/include/uapi/linux/virtio_config.h
@@ -56,6 +56,9 @@
  * suppressed them? */
 #define VIRTIO_F_NOTIFY_ON_EMPTY	24
 
+/* Driver supports the VIRTIO_CONFIG_S_NEEDS_RESET command */
+#define VIRTIO_F_CAN_RESET		25
+
 /* Can the device handle any descriptor layout? */
 #define VIRTIO_F_ANY_LAYOUT		27
 #endif /* VIRTIO_CONFIG_NO_LEGACY */
-- 
2.14.1.342.g6490525c54-goog

Re: [PATCH RFC 2/2] virtio_net: enable probing for NEEDS_RESET support

[Michael S. Tsirkin]

On Tue, Aug 29, 2017 at 04:07:59PM -0400, Willem de Bruijn wrote:
> From: Willem de Bruijn <willemb@google.com>
> 
> Implement a mechanism to signal that a virtio device implements the
> VIRTIO_CONFIG_S_NEEDS_RESET command.
> 
> Testing for VIRTIO_CONFIG_S_NEEDS_RESET support by issuing the request
> and verifying the reset state would require an expensive state change.
> 
> To avoid that, add a status bit to the feature register used during
> feature negotiation between host and guest.
> 
> Set the bit for virtio-net, which supports the feature.
> 
> Signed-off-by: Willem de Bruijn <willemb@google.com>

All virtio 1 devices have the reset feature so maybe guest does
not need this flag. Does device need it? Does device really
care that guest can't recover?

> ---
>  drivers/net/virtio_net.c           | 2 ++
>  include/uapi/linux/virtio_config.h | 3 +++
>  2 files changed, 5 insertions(+)
> 
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index 5e349226f7c1..15483a982106 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -2801,12 +2801,14 @@ static struct virtio_device_id id_table[] = {
>  
>  static unsigned int features[] = {
>  	VIRTNET_FEATURES,
> +	VIRTIO_F_CAN_RESET,
>  };
>  
>  static unsigned int features_legacy[] = {
>  	VIRTNET_FEATURES,
>  	VIRTIO_NET_F_GSO,
>  	VIRTIO_F_ANY_LAYOUT,
> +	VIRTIO_F_CAN_RESET,
>  };
>  
>  static struct virtio_driver virtio_net_driver = {
> diff --git a/include/uapi/linux/virtio_config.h b/include/uapi/linux/virtio_config.h
> index 308e2096291f..726be44a04d3 100644
> --- a/include/uapi/linux/virtio_config.h
> +++ b/include/uapi/linux/virtio_config.h
> @@ -56,6 +56,9 @@
>   * suppressed them? */
>  #define VIRTIO_F_NOTIFY_ON_EMPTY	24
>  
> +/* Driver supports the VIRTIO_CONFIG_S_NEEDS_RESET command */
> +#define VIRTIO_F_CAN_RESET		25
> +
>  /* Can the device handle any descriptor layout? */
>  #define VIRTIO_F_ANY_LAYOUT		27
>  #endif /* VIRTIO_CONFIG_NO_LEGACY */
> -- 
> 2.14.1.342.g6490525c54-goog

Re: [PATCH RFC 2/2] virtio_net: enable probing for NEEDS_RESET support

[Willem de Bruijn]

On Tue, Aug 29, 2017 at 4:16 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> On Tue, Aug 29, 2017 at 04:07:59PM -0400, Willem de Bruijn wrote:
>> From: Willem de Bruijn <willemb@google.com>
>>
>> Implement a mechanism to signal that a virtio device implements the
>> VIRTIO_CONFIG_S_NEEDS_RESET command.
>>
>> Testing for VIRTIO_CONFIG_S_NEEDS_RESET support by issuing the request
>> and verifying the reset state would require an expensive state change.
>>
>> To avoid that, add a status bit to the feature register used during
>> feature negotiation between host and guest.
>>
>> Set the bit for virtio-net, which supports the feature.
>>
>> Signed-off-by: Willem de Bruijn <willemb@google.com>
>
> All virtio 1 devices have the reset feature

I don't quite follow. No device drivers implement this request currently?

> so maybe guest does
> not need this flag. Does device need it? Does device really
> care that guest can't recover?

If all device drivers support it, then the flag is not needed.

But as long as legacy device drivers can exist that do not support
this feature, it has to have a way of differentiating between the two.

Re: [PATCH RFC 2/2] virtio_net: enable probing for NEEDS_RESET support

[Michael S. Tsirkin]

On Tue, Aug 29, 2017 at 04:27:41PM -0400, Willem de Bruijn wrote:
> On Tue, Aug 29, 2017 at 4:16 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > On Tue, Aug 29, 2017 at 04:07:59PM -0400, Willem de Bruijn wrote:
> >> From: Willem de Bruijn <willemb@google.com>
> >>
> >> Implement a mechanism to signal that a virtio device implements the
> >> VIRTIO_CONFIG_S_NEEDS_RESET command.
> >>
> >> Testing for VIRTIO_CONFIG_S_NEEDS_RESET support by issuing the request
> >> and verifying the reset state would require an expensive state change.
> >>
> >> To avoid that, add a status bit to the feature register used during
> >> feature negotiation between host and guest.
> >>
> >> Set the bit for virtio-net, which supports the feature.
> >>
> >> Signed-off-by: Willem de Bruijn <willemb@google.com>
> >
> > All virtio 1 devices have the reset feature
> 
> I don't quite follow. No device drivers implement this request currently?

Depends. Spec 1.0 describes the bit and that driver can respond
by reseting the device. You seem to do something else
in this patchset, but as designed in 1.0 it does not seem to need
a feature bit.

> > so maybe guest does
> > not need this flag. Does device need it? Does device really
> > care that guest can't recover?
> 
> If all device drivers support it, then the flag is not needed.
> 
> But as long as legacy device drivers can exist that do not support
> this feature, it has to have a way of differentiating between the two.

Why? Device won't set this unless it's in a bad state. In that case,
setting the bit is harmless even if drivers ignore it.

-- 
MST

Re: [PATCH RFC 2/2] virtio_net: enable probing for NEEDS_RESET support

[Willem de Bruijn]

+ virtio-dev

On Tue, Aug 29, 2017 at 4:38 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> On Tue, Aug 29, 2017 at 04:27:41PM -0400, Willem de Bruijn wrote:
>> On Tue, Aug 29, 2017 at 4:16 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
>> > On Tue, Aug 29, 2017 at 04:07:59PM -0400, Willem de Bruijn wrote:
>> >> From: Willem de Bruijn <willemb@google.com>
>> >>
>> >> Implement a mechanism to signal that a virtio device implements the
>> >> VIRTIO_CONFIG_S_NEEDS_RESET command.
>> >>
>> >> Testing for VIRTIO_CONFIG_S_NEEDS_RESET support by issuing the request
>> >> and verifying the reset state would require an expensive state change.
>> >>
>> >> To avoid that, add a status bit to the feature register used during
>> >> feature negotiation between host and guest.
>> >>
>> >> Set the bit for virtio-net, which supports the feature.
>> >>
>> >> Signed-off-by: Willem de Bruijn <willemb@google.com>
>> >
>> > All virtio 1 devices have the reset feature
>>
>> I don't quite follow. No device drivers implement this request currently?
>
> Depends. Spec 1.0 describes the bit and that driver can respond
> by reseting the device. You seem to do something else
> in this patchset, but as designed in 1.0 it does not seem to need
> a feature bit.

I see. So support is designed to be best effort?

The feature bit is only needed if driver support is optional.

The ack response is necessary if the device acts differently
depending on whether the reset happened. The device has
to reset its local state, too, so I think that this is needed.


>> > so maybe guest does
>> > not need this flag. Does device need it? Does device really
>> > care that guest can't recover?
>>
>> If all device drivers support it, then the flag is not needed.
>>
>> But as long as legacy device drivers can exist that do not support
>> this feature, it has to have a way of differentiating between the two.
>
> Why? Device won't set this unless it's in a bad state. In that case,
> setting the bit is harmless even if drivers ignore it.

The goal is for the device to be able to rely on the driver reset to get
to a good state even if it gets it into a bad state.

That allows it to implement optimizations that could get it into that bad
state.

In particular, in the edge case where the device performs backpressure,
takes the descriptor out of the avail ring and does not immediately post
it to the used ring.

A reset will make the guest free all delayed packets and treat any
unsent and unacknowledged as network drops. This allows the
device to indeed drop long delayed packets when they eventually
surface (e.g., leave a qdisc queue). This is of course not safe with
zerocopy packets.

Re: [PATCH RFC 2/2] virtio_net: enable probing for NEEDS_RESET support

[Michael S. Tsirkin]

On Tue, Aug 29, 2017 at 05:02:29PM -0400, Willem de Bruijn wrote:
> + virtio-dev
> 
> On Tue, Aug 29, 2017 at 4:38 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> > On Tue, Aug 29, 2017 at 04:27:41PM -0400, Willem de Bruijn wrote:
> >> On Tue, Aug 29, 2017 at 4:16 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> >> > On Tue, Aug 29, 2017 at 04:07:59PM -0400, Willem de Bruijn wrote:
> >> >> From: Willem de Bruijn <willemb@google.com>
> >> >>
> >> >> Implement a mechanism to signal that a virtio device implements the
> >> >> VIRTIO_CONFIG_S_NEEDS_RESET command.
> >> >>
> >> >> Testing for VIRTIO_CONFIG_S_NEEDS_RESET support by issuing the request
> >> >> and verifying the reset state would require an expensive state change.
> >> >>
> >> >> To avoid that, add a status bit to the feature register used during
> >> >> feature negotiation between host and guest.
> >> >>
> >> >> Set the bit for virtio-net, which supports the feature.
> >> >>
> >> >> Signed-off-by: Willem de Bruijn <willemb@google.com>
> >> >
> >> > All virtio 1 devices have the reset feature
> >>
> >> I don't quite follow. No device drivers implement this request currently?
> >
> > Depends. Spec 1.0 describes the bit and that driver can respond
> > by reseting the device. You seem to do something else
> > in this patchset, but as designed in 1.0 it does not seem to need
> > a feature bit.
> 
> I see. So support is designed to be best effort?
> 
> The feature bit is only needed if driver support is optional.
> 
> The ack response is necessary if the device acts differently
> depending on whether the reset happened. The device has
> to reset its local state, too, so I think that this is needed.

That reset should only happen when guest driver resets the device.
And spec already has a mechanism for that anyway.

> 
> >> > so maybe guest does
> >> > not need this flag. Does device need it? Does device really
> >> > care that guest can't recover?
> >>
> >> If all device drivers support it, then the flag is not needed.
> >>
> >> But as long as legacy device drivers can exist that do not support
> >> this feature, it has to have a way of differentiating between the two.
> >
> > Why? Device won't set this unless it's in a bad state. In that case,
> > setting the bit is harmless even if drivers ignore it.
> 
> The goal is for the device to be able to rely on the driver reset to get
> to a good state even if it gets it into a bad state.
> 
> That allows it to implement optimizations that could get it into that bad
> state.

I see. I don't think this is what the need reset was designed for.

We can extend it to cover this case but let's add a bit more
documentation then.

And in particular won't it be better if we could just reset one ring,
and not the whole device state? This might be nicer so flows on other
rings aren't disrupted.

> 
> In particular, in the edge case where the device performs backpressure,
> takes the descriptor out of the avail ring and does not immediately post
> it to the used ring.
> 
> A reset will make the guest free all delayed packets and treat any
> unsent and unacknowledged as network drops. This allows the
> device to indeed drop long delayed packets when they eventually
> surface (e.g., leave a qdisc queue).

In this particular case, won't it be easier for device to just
report all packets as used, without involving the guest?

> This is of course not safe with
> zerocopy packets.


I wonder if we can teach kernel to drop zero copy packets too.

-- 
MST

Re: [PATCH RFC 2/2] virtio_net: enable probing for NEEDS_RESET support

[Willem de Bruijn]

On Tue, Aug 29, 2017 at 5:13 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> On Tue, Aug 29, 2017 at 05:02:29PM -0400, Willem de Bruijn wrote:
>> + virtio-dev
>>
>> On Tue, Aug 29, 2017 at 4:38 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
>> > On Tue, Aug 29, 2017 at 04:27:41PM -0400, Willem de Bruijn wrote:
>> >> On Tue, Aug 29, 2017 at 4:16 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
>> >> > On Tue, Aug 29, 2017 at 04:07:59PM -0400, Willem de Bruijn wrote:
>> >> >> From: Willem de Bruijn <willemb@google.com>
>> >> >>
>> >> >> Implement a mechanism to signal that a virtio device implements the
>> >> >> VIRTIO_CONFIG_S_NEEDS_RESET command.
>> >> >>
>> >> >> Testing for VIRTIO_CONFIG_S_NEEDS_RESET support by issuing the request
>> >> >> and verifying the reset state would require an expensive state change.
>> >> >>
>> >> >> To avoid that, add a status bit to the feature register used during
>> >> >> feature negotiation between host and guest.
>> >> >>
>> >> >> Set the bit for virtio-net, which supports the feature.
>> >> >>
>> >> >> Signed-off-by: Willem de Bruijn <willemb@google.com>
>> >> >
>> >> > All virtio 1 devices have the reset feature
>> >>
>> >> I don't quite follow. No device drivers implement this request currently?
>> >
>> > Depends. Spec 1.0 describes the bit and that driver can respond
>> > by reseting the device. You seem to do something else
>> > in this patchset, but as designed in 1.0 it does not seem to need
>> > a feature bit.
>>
>> I see. So support is designed to be best effort?
>>
>> The feature bit is only needed if driver support is optional.
>>
>> The ack response is necessary if the device acts differently
>> depending on whether the reset happened. The device has
>> to reset its local state, too, so I think that this is needed.
>
> That reset should only happen when guest driver resets the device.
> And spec already has a mechanism for that anyway.

And the device can read the ring state to see whether it has reset,
perhaps.

>>
>> >> > so maybe guest does
>> >> > not need this flag. Does device need it? Does device really
>> >> > care that guest can't recover?
>> >>
>> >> If all device drivers support it, then the flag is not needed.
>> >>
>> >> But as long as legacy device drivers can exist that do not support
>> >> this feature, it has to have a way of differentiating between the two.
>> >
>> > Why? Device won't set this unless it's in a bad state. In that case,
>> > setting the bit is harmless even if drivers ignore it.
>>
>> The goal is for the device to be able to rely on the driver reset to get
>> to a good state even if it gets it into a bad state.
>>
>> That allows it to implement optimizations that could get it into that bad
>> state.
>
> I see. I don't think this is what the need reset was designed for.
>
> We can extend it to cover this case but let's add a bit more
> documentation then.

Okay.

> And in particular won't it be better if we could just reset one ring,
> and not the whole device state? This might be nicer so flows on other
> rings aren't disrupted.

Indeed. But that would require a different request, then? It also
depends on the use case. A full device reset is a big hammer,
but if used only to get out of rare edge cases, it is good enough.

>>
>> In particular, in the edge case where the device performs backpressure,
>> takes the descriptor out of the avail ring and does not immediately post
>> it to the used ring.
>>
>> A reset will make the guest free all delayed packets and treat any
>> unsent and unacknowledged as network drops. This allows the
>> device to indeed drop long delayed packets when they eventually
>> surface (e.g., leave a qdisc queue).
>
> In this particular case, won't it be easier for device to just
> report all packets as used, without involving the guest?

When the device can just iterate over the outstanding packet, yeah,
that's actually simpler.

>> This is of course not safe with
>> zerocopy packets.
>
>
> I wonder if we can teach kernel to drop zero copy packets too.

Your point about changing frags[] underneath a cloned skb really does
make that hard. We might be able to mitigate individual specific cases
of high latency, such as TC queue occupancy.

>
> --
> MST

Re: [PATCH RFC 1/2] virtio_net: implement VIRTIO_CONFIG_S_NEEDS_RESET

[Michael S. Tsirkin]

On Tue, Aug 29, 2017 at 04:07:58PM -0400, Willem de Bruijn wrote:
> From: Willem de Bruijn <willemb@google.com>
> 
> Implement the reset communication request defined in the VIRTIO 1.0
> specification and introduces in Linux in commit c00bbcf862896 ("virtio:
> add VIRTIO_CONFIG_S_NEEDS_RESET device status bit").
> 
> Since that patch, the virtio-net driver has added a virtnet_reset
> function that implements the requested behavior through calls to the
> power management freeze and restore functions.
> 
> That function has recently been reverted when its sole caller was
> updated. Bring it back and listen for the request from the host on
> the config channel.
> 
> Implement the feature analogous to other config requests. In
> particular, acknowledge the request in the same manner as the
> NET_S_ANNOUNCE link announce request, by responding with a new
> VIRTIO_NET_CTRL_${TYPE} command. On reception, the host must check
> the config status register for success or failure.


Pls make it clearer why do you need these interface extensions.

> The existing freeze handler verifies that no config changes are
> running concurrently. Elide this check for reset. The request is
> always handled from the config workqueue. No other config requests
> can be active or scheduled concurrently on vi->config.

You need to prevent packet TX from being in progress.

> 
> Signed-off-by: Willem de Bruijn <willemb@google.com>
> ---
>  drivers/net/virtio_net.c        | 69 +++++++++++++++++++++++++++++++++++------
>  include/uapi/linux/virtio_net.h |  4 +++

virtio dev or another virtio TC list must be Cc'd on any proposed API changes.


>  2 files changed, 64 insertions(+), 9 deletions(-)
> 
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index 52ae78ca3d38..5e349226f7c1 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -1458,12 +1458,11 @@ static void virtnet_netpoll(struct net_device *dev)
>  }
>  #endif
>  
> -static void virtnet_ack_link_announce(struct virtnet_info *vi)
> +static void virtnet_ack(struct virtnet_info *vi, u8 class, u8 cmd)
>  {
>  	rtnl_lock();
> -	if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
> -				  VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
> -		dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
> +	if (!virtnet_send_command(vi, class, cmd, NULL))
> +		dev_warn(&vi->dev->dev, "Failed to ack %u.%u\n", class, cmd);
>  	rtnl_unlock();
>  }
>  
> @@ -1857,13 +1856,16 @@ static const struct ethtool_ops virtnet_ethtool_ops = {
>  	.set_link_ksettings = virtnet_set_link_ksettings,
>  };
>  
> -static void virtnet_freeze_down(struct virtio_device *vdev)
> +static void virtnet_freeze_down(struct virtio_device *vdev, bool in_reset)
>  {
>  	struct virtnet_info *vi = vdev->priv;
>  	int i;
>  
> -	/* Make sure no work handler is accessing the device */
> -	flush_work(&vi->config_work);
> +	/* Make sure no work handler is accessing the device,
> +	 * unless this call is made from the reset work handler itself.
> +	 */
> +	if (!in_reset)
> +		flush_work(&vi->config_work);
>  
>  	netif_device_detach(vi->dev);
>  	netif_tx_disable(vi->dev);
> @@ -1878,6 +1880,7 @@ static void virtnet_freeze_down(struct virtio_device *vdev)
>  }
>  
>  static int init_vqs(struct virtnet_info *vi);
> +static void remove_vq_common(struct virtnet_info *vi);
>  
>  static int virtnet_restore_up(struct virtio_device *vdev)
>  {
> @@ -1906,6 +1909,45 @@ static int virtnet_restore_up(struct virtio_device *vdev)
>  	return err;
>  }
>  
> +static int virtnet_reset(struct virtnet_info *vi)
> +{
> +	struct virtio_device *dev = vi->vdev;
> +	bool failed;
> +	int ret;
> +
> +	virtio_config_disable(dev);
> +	failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
> +	virtnet_freeze_down(dev, true);
> +	remove_vq_common(vi);
> +
> +	virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
> +	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
> +
> +	/* Restore the failed status (see virtio_device_restore). */
> +	if (failed)
> +		virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> +
> +	ret = virtio_finalize_features(dev);
> +	if (ret)
> +		goto err;
> +
> +	ret = virtnet_restore_up(dev);
> +	if (ret)
> +		goto err;
> +
> +	ret = virtnet_set_queues(vi, vi->curr_queue_pairs);
> +	if (ret)
> +		goto err;
> +
> +	virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
> +	virtio_config_enable(dev);
> +	return 0;
> +
> +err:
> +	virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
> +	return ret;
> +}
> +
>  static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
>  {
>  	struct scatterlist sg;
> @@ -2085,7 +2127,16 @@ static void virtnet_config_changed_work(struct work_struct *work)
>  
>  	if (v & VIRTIO_NET_S_ANNOUNCE) {
>  		netdev_notify_peers(vi->dev);
> -		virtnet_ack_link_announce(vi);
> +		virtnet_ack(vi, VIRTIO_NET_CTRL_ANNOUNCE,
> +			    VIRTIO_NET_CTRL_ANNOUNCE_ACK);
> +	}
> +
> +	if (vi->vdev->config->get_status(vi->vdev) &
> +	    VIRTIO_CONFIG_S_NEEDS_RESET) {
> +		virtnet_reset(vi);
> +		virtnet_ack(vi, VIRTIO_NET_CTRL_RESET,
> +			    VIRTIO_NET_CTRL_RESET_ACK);
> +
>  	}
>  
>  	/* Ignore unknown (future) status bits */
> @@ -2708,7 +2759,7 @@ static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
>  	struct virtnet_info *vi = vdev->priv;
>  
>  	virtnet_cpu_notif_remove(vi);
> -	virtnet_freeze_down(vdev);
> +	virtnet_freeze_down(vdev, false);
>  	remove_vq_common(vi);
>  
>  	return 0;
> diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h
> index fc353b518288..188fdc528f13 100644
> --- a/include/uapi/linux/virtio_net.h
> +++ b/include/uapi/linux/virtio_net.h
> @@ -245,4 +245,8 @@ struct virtio_net_ctrl_mq {
>  #define VIRTIO_NET_CTRL_GUEST_OFFLOADS   5
>  #define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET        0
>  
> +/* Signal that the driver received and executed the reset command. */
> +#define VIRTIO_NET_CTRL_RESET			6
> +#define VIRTIO_NET_CTRL_RESET_ACK		0
> +
>  #endif /* _UAPI_LINUX_VIRTIO_NET_H */
> -- 
> 2.14.1.342.g6490525c54-goog

Re: [PATCH RFC 1/2] virtio_net: implement VIRTIO_CONFIG_S_NEEDS_RESET

[Willem de Bruijn]

On Tue, Aug 29, 2017 at 4:20 PM, Michael S. Tsirkin <mst@redhat.com> wrote:
> On Tue, Aug 29, 2017 at 04:07:58PM -0400, Willem de Bruijn wrote:
>> From: Willem de Bruijn <willemb@google.com>
>>
>> Implement the reset communication request defined in the VIRTIO 1.0
>> specification and introduces in Linux in commit c00bbcf862896 ("virtio:
>> add VIRTIO_CONFIG_S_NEEDS_RESET device status bit").
>>
>> Since that patch, the virtio-net driver has added a virtnet_reset
>> function that implements the requested behavior through calls to the
>> power management freeze and restore functions.
>>
>> That function has recently been reverted when its sole caller was
>> updated. Bring it back and listen for the request from the host on
>> the config channel.
>>
>> Implement the feature analogous to other config requests. In
>> particular, acknowledge the request in the same manner as the
>> NET_S_ANNOUNCE link announce request, by responding with a new
>> VIRTIO_NET_CTRL_${TYPE} command. On reception, the host must check
>> the config status register for success or failure.
>
>
> Pls make it clearer why do you need these interface extensions.
>
>> The existing freeze handler verifies that no config changes are
>> running concurrently. Elide this check for reset. The request is
>> always handled from the config workqueue. No other config requests
>> can be active or scheduled concurrently on vi->config.
>
> You need to prevent packet TX from being in progress.

I had another look at this.

As of commit 713a98d90c5e ("virtio-net: serialize tx routine during reset")
virtnet_freeze_down calls synchronize_net() after stopping the queues
to quiesce the device before any further actions. This should suffice for
virtnet_reset, too.

The control path can indeed be much simpler than my initial patchset.
The host can read vdev->status to observe when the reset went through.
It adds flag VIRTIO_CONFIG_S_NEEDS_RESET to request the reset.
This flag is cleared by the operation itself.

>
>>
>> Signed-off-by: Willem de Bruijn <willemb@google.com>
>> ---
>>  drivers/net/virtio_net.c        | 69 +++++++++++++++++++++++++++++++++++------
>>  include/uapi/linux/virtio_net.h |  4 +++
>
> virtio dev or another virtio TC list must be Cc'd on any proposed API changes.
>
>
>>  2 files changed, 64 insertions(+), 9 deletions(-)
>>
>> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
>> index 52ae78ca3d38..5e349226f7c1 100644
>> --- a/drivers/net/virtio_net.c
>> +++ b/drivers/net/virtio_net.c
>> @@ -1458,12 +1458,11 @@ static void virtnet_netpoll(struct net_device *dev)
>>  }
>>  #endif
>>
>> -static void virtnet_ack_link_announce(struct virtnet_info *vi)
>> +static void virtnet_ack(struct virtnet_info *vi, u8 class, u8 cmd)
>>  {
>>       rtnl_lock();
>> -     if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE,
>> -                               VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL))
>> -             dev_warn(&vi->dev->dev, "Failed to ack link announce.\n");
>> +     if (!virtnet_send_command(vi, class, cmd, NULL))
>> +             dev_warn(&vi->dev->dev, "Failed to ack %u.%u\n", class, cmd);
>>       rtnl_unlock();
>>  }
>>
>> @@ -1857,13 +1856,16 @@ static const struct ethtool_ops virtnet_ethtool_ops = {
>>       .set_link_ksettings = virtnet_set_link_ksettings,
>>  };
>>
>> -static void virtnet_freeze_down(struct virtio_device *vdev)
>> +static void virtnet_freeze_down(struct virtio_device *vdev, bool in_reset)
>>  {
>>       struct virtnet_info *vi = vdev->priv;
>>       int i;
>>
>> -     /* Make sure no work handler is accessing the device */
>> -     flush_work(&vi->config_work);
>> +     /* Make sure no work handler is accessing the device,
>> +      * unless this call is made from the reset work handler itself.
>> +      */
>> +     if (!in_reset)
>> +             flush_work(&vi->config_work);
>>
>>       netif_device_detach(vi->dev);
>>       netif_tx_disable(vi->dev);
>> @@ -1878,6 +1880,7 @@ static void virtnet_freeze_down(struct virtio_device *vdev)
>>  }
>>
>>  static int init_vqs(struct virtnet_info *vi);
>> +static void remove_vq_common(struct virtnet_info *vi);
>>
>>  static int virtnet_restore_up(struct virtio_device *vdev)
>>  {
>> @@ -1906,6 +1909,45 @@ static int virtnet_restore_up(struct virtio_device *vdev)
>>       return err;
>>  }
>>
>> +static int virtnet_reset(struct virtnet_info *vi)
>> +{
>> +     struct virtio_device *dev = vi->vdev;
>> +     bool failed;
>> +     int ret;
>> +
>> +     virtio_config_disable(dev);
>> +     failed = dev->config->get_status(dev) & VIRTIO_CONFIG_S_FAILED;
>> +     virtnet_freeze_down(dev, true);
>> +     remove_vq_common(vi);
>> +
>> +     virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE);
>> +     virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER);
>> +
>> +     /* Restore the failed status (see virtio_device_restore). */
>> +     if (failed)
>> +             virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>> +
>> +     ret = virtio_finalize_features(dev);
>> +     if (ret)
>> +             goto err;
>> +
>> +     ret = virtnet_restore_up(dev);
>> +     if (ret)
>> +             goto err;
>> +
>> +     ret = virtnet_set_queues(vi, vi->curr_queue_pairs);
>> +     if (ret)
>> +             goto err;
>> +
>> +     virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
>> +     virtio_config_enable(dev);
>> +     return 0;
>> +
>> +err:
>> +     virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED);
>> +     return ret;
>> +}
>> +
>>  static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads)
>>  {
>>       struct scatterlist sg;
>> @@ -2085,7 +2127,16 @@ static void virtnet_config_changed_work(struct work_struct *work)
>>
>>       if (v & VIRTIO_NET_S_ANNOUNCE) {
>>               netdev_notify_peers(vi->dev);
>> -             virtnet_ack_link_announce(vi);
>> +             virtnet_ack(vi, VIRTIO_NET_CTRL_ANNOUNCE,
>> +                         VIRTIO_NET_CTRL_ANNOUNCE_ACK);
>> +     }
>> +
>> +     if (vi->vdev->config->get_status(vi->vdev) &
>> +         VIRTIO_CONFIG_S_NEEDS_RESET) {
>> +             virtnet_reset(vi);
>> +             virtnet_ack(vi, VIRTIO_NET_CTRL_RESET,
>> +                         VIRTIO_NET_CTRL_RESET_ACK);
>> +
>>       }
>>
>>       /* Ignore unknown (future) status bits */
>> @@ -2708,7 +2759,7 @@ static __maybe_unused int virtnet_freeze(struct virtio_device *vdev)
>>       struct virtnet_info *vi = vdev->priv;
>>
>>       virtnet_cpu_notif_remove(vi);
>> -     virtnet_freeze_down(vdev);
>> +     virtnet_freeze_down(vdev, false);
>>       remove_vq_common(vi);
>>
>>       return 0;
>> diff --git a/include/uapi/linux/virtio_net.h b/include/uapi/linux/virtio_net.h
>> index fc353b518288..188fdc528f13 100644
>> --- a/include/uapi/linux/virtio_net.h
>> +++ b/include/uapi/linux/virtio_net.h
>> @@ -245,4 +245,8 @@ struct virtio_net_ctrl_mq {
>>  #define VIRTIO_NET_CTRL_GUEST_OFFLOADS   5
>>  #define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET        0
>>
>> +/* Signal that the driver received and executed the reset command. */
>> +#define VIRTIO_NET_CTRL_RESET                        6
>> +#define VIRTIO_NET_CTRL_RESET_ACK            0
>> +
>>  #endif /* _UAPI_LINUX_VIRTIO_NET_H */
>> --
>> 2.14.1.342.g6490525c54-goog