Re: [PATCH 2/3] virtio_net: introduce an RX queue watchdog for stuck detection

["Michael S. Tsirkin" <mst@redhat.com>]

On Mon, Nov 09, 2026 at 05:43:09PM +0800, Longjun Tang wrote:
> From: Longjun Tang <tanglongjun@kylinos.cn>
> 
> The backend raises an interrupt after adding buffers to the used ring,
> and the hypervisor injects it. If one is lost (e.g., a transient KVM
> failure), NAPI will not be scheduled and the driver stops consuming
> buffers. The ring eventually fills up, the backend stops notifying, and
> the queue is left permanently stuck.
> 
> Add a per-device watchdog that scans the RX queues once per second and
> detects a queue that has a non-zero backlog while making no consumption
> progress and receiving no new interrupt for rx_watchdog_timeo seconds
> (default 5, 0 disables it). On detection, reschedule NAPI.
> 
> Signed-off-by: Longjun Tang <tanglongjun@kylinos.cn>
> ---
>  drivers/net/virtio_net.c | 104 +++++++++++++++++++++++++++++++++++++++
>  1 file changed, 104 insertions(+)
> 
> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
> index dd2c41ef6fbe..ec7c248080c5 100644
> --- a/drivers/net/virtio_net.c
> +++ b/drivers/net/virtio_net.c
> @@ -36,6 +36,10 @@ module_param(csum, bool, 0444);
>  module_param(gso, bool, 0444);
>  module_param(napi_tx, bool, 0644);
>  
> +/* RX queue stall timeout in seconds; 0 disables the watchdog. */
> +static unsigned int rx_watchdog_timeo = 5;
> +module_param(rx_watchdog_timeo, uint, 0644);
> +
>  #define VIRTIO_OFFLOAD_MAP_MIN	46
>  #define VIRTIO_OFFLOAD_MAP_MAX	47
>  #define VIRTIO_FEATURES_MAP_MIN	65

No module params pls. This is not a sane UAPI.

> @@ -318,6 +322,13 @@ struct send_queue {
>  	dma_addr_t xsk_hdr_dma_addr;
>  };
>  
> +struct virtnet_rq_watchdog {
> +	u16 last_used;
> +	u16 calls;
> +	unsigned long jiffies;
> +	bool stalled;
> +};
> +
>  /* Internal representation of a receive virtqueue */
>  struct receive_queue {
>  	/* Virtqueue associated with this receive_queue */
> @@ -332,6 +343,9 @@ struct receive_queue {
>  	/* The number of rx notifications */
>  	u16 calls;
>  
> +	/* RX watchdog state. */
> +	struct virtnet_rq_watchdog watchdog;
> +
>  	/* Is dynamic interrupt moderation enabled? */
>  	bool dim_enabled;
>  
> @@ -443,6 +457,9 @@ struct virtnet_info {
>  	/* Work struct for setting rx mode */
>  	struct work_struct rx_mode_work;
>  
> +	/* RX watchdog timer for stuck detection */
> +	struct timer_list rx_watchdog;
> +
>  	/* Does the affinity hint is set for virtqueues? */
>  	bool affinity_hint_set;
>  
> @@ -3067,6 +3084,80 @@ static int virtnet_poll(struct napi_struct *napi, int budget)
>  	return received;
>  }
>  
> +/*
> + * An RX queue is stalled if it has a backlog but makes no progress and
> + * gets no new interrupt for rx_watchdog_timeo seconds, usually because
> + * an interrupt was lost.  Mark it stalled and reschedule NAPI to drain
> + * the ring, refill the buffers.
> + */
> +static void virtnet_rx_watchdog(struct timer_list *t)
> +{
> +	struct virtnet_info *vi = timer_container_of(vi, t, rx_watchdog);
> +	unsigned long timeout = rx_watchdog_timeo * HZ;
> +	int i;
> +
> +	if (!rx_watchdog_timeo)
> +		return;
> +
> +	for (i = 0; i < vi->curr_queue_pairs; i++) {
> +		struct receive_queue *rq = &vi->rq[i];
> +		u16 last_used = virtqueue_get_last_used_idx(rq->vq);
> +		u16 calls = rq->calls;
> +		bool backlog = virtqueue_poll(rq->vq, last_used);
> +		bool progress = last_used != rq->watchdog.last_used;
> +
> +		if (!backlog || progress || calls != rq->watchdog.calls) {
> +			rq->watchdog.last_used = last_used;
> +			rq->watchdog.calls = calls;
> +			rq->watchdog.jiffies = jiffies;
> +			continue;
> +		}
> +
> +		if (time_after(jiffies, rq->watchdog.jiffies + timeout)) {
> +			unsigned int stall_ms =
> +				jiffies_to_msecs(jiffies - rq->watchdog.jiffies);
> +
> +			netdev_warn(vi->dev,
> +				    "RX queue %u stalled for %u ms, rescheduling NAPI\n",
> +				    i, stall_ms);
> +
> +			rq->watchdog.stalled = true;
> +			virtqueue_napi_schedule(&rq->napi, rq->vq);
> +
> +			/* Rate-limit to one event per timeout. */
> +			rq->watchdog.jiffies = jiffies;
> +		}
> +	}
> +
> +	mod_timer(&vi->rx_watchdog, jiffies + HZ);
> +}
> +
> +static void virtnet_rx_watchdog_init_queue(struct receive_queue *rq)
> +{
> +	rq->watchdog.last_used = virtqueue_get_last_used_idx(rq->vq);
> +	rq->watchdog.calls = rq->calls;
> +	rq->watchdog.jiffies = jiffies;
> +	rq->watchdog.stalled = false;
> +}
> +
> +static void virtnet_rx_watchdog_start(struct virtnet_info *vi)
> +{
> +	int i;
> +
> +	if (!rx_watchdog_timeo)
> +		return;
> +
> +	for (i = 0; i < vi->curr_queue_pairs; i++)
> +		virtnet_rx_watchdog_init_queue(&vi->rq[i]);
> +
> +	mod_timer(&vi->rx_watchdog, jiffies + HZ);
> +}
> +
> +static void virtnet_rx_watchdog_stop(struct virtnet_info *vi)
> +{
> +	timer_delete_sync(&vi->rx_watchdog);
> +}
> +
>  static void virtnet_disable_queue_pair(struct virtnet_info *vi, int qp_index)
>  {
>  	virtnet_napi_tx_disable(&vi->sq[qp_index]);
> @@ -3230,6 +3321,8 @@ static int virtnet_open(struct net_device *dev)
>  		netif_carrier_on(dev);
>  	}
>  
> +	virtnet_rx_watchdog_start(vi);
> +
>  	return 0;
>  
>  err_enable_qp:
> @@ -3759,6 +3852,7 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
>  	struct virtio_net_rss_config_trailer old_rss_trailer;
>  	struct net_device *dev = vi->dev;
>  	struct scatterlist sg;
> +	u16 old_queue_pairs = vi->curr_queue_pairs;
>  
>  	if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
>  		return 0;
> @@ -3816,7 +3910,14 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
>  	if (vi->has_rss)
>  		vi->rss_trailer.max_tx_vq = cpu_to_le16(queue_pairs);
>  succ:
> +	if (dev->flags & IFF_UP) {
> +		/* Init new queues */
> +		for (int i = old_queue_pairs; i < queue_pairs; i++)
> +			virtnet_rx_watchdog_init_queue(&vi->rq[i]);
> +	}
> +
>  	vi->curr_queue_pairs = queue_pairs;
> +
>  	if (dev->flags & IFF_UP) {
>  		local_bh_disable();
>  		for (int i = 0; i < vi->curr_queue_pairs; ++i)
> @@ -3832,6 +3933,8 @@ static int virtnet_close(struct net_device *dev)
>  	struct virtnet_info *vi = netdev_priv(dev);
>  	int i;
>  
> +	virtnet_rx_watchdog_stop(vi);
> +
>  	/* Prevent the config change callback from changing carrier
>  	 * after close
>  	 */
> @@ -6900,6 +7003,7 @@ static int virtnet_probe(struct virtio_device *vdev)
>  	INIT_WORK(&vi->config_work, virtnet_config_changed_work);
>  	INIT_WORK(&vi->rx_mode_work, virtnet_rx_mode_work);
>  	disable_work(&vi->rx_mode_work);
> +	timer_setup(&vi->rx_watchdog, virtnet_rx_watchdog, 0);
>  
>  	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) {
>  		vi->mergeable_rx_bufs = true;
> -- 
> 2.51.0