netdev.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: "Lange Tang" <lange_tang@163.com>
To: "mst@redhat.com" <mst@redhat.com>
Cc: "Jakub Kicinski" <kuba@kernel.org>,
	"Eugenio Perez" <eperezma@redhat.com>,
	"jasowangio@gmail.com" <jasowangio@gmail.com>,
	"xuanzhuo@linux.alibaba.com" <xuanzhuo@linux.alibaba.com>,
	"virtualization@lists.linux.dev" <virtualization@lists.linux.dev>,
	"netdev@vger.kernel.org" <netdev@vger.kernel.org>,
	"Tang Longjun" <tanglongjun@kylinos.cn>
Subject: Re:Re: [PATCH 2/3] virtio_net: introduce an RX queue watchdog for stuck detection
Date: Fri, 20 Nov 2026 15:14:28 +0800 (CST)	[thread overview]
Message-ID: <7158076d.5a7a.1a1f7ce82d5.Coremail.lange_tang@163.com> (raw)
In-Reply-To: <20261112175123-mutt-send-email-mst@kernel.org>

At 2026-11-13 06:51:46, "Michael S. Tsirkin" <mst@redhat.com> wrote:
>On Mon, Nov 09, 2026 at 05:43:09PM +0800, Longjun Tang wrote:
>> From: Longjun Tang <tanglongjun@kylinos.cn>
>> 
>> The backend raises an interrupt after adding buffers to the used ring,
>> and the hypervisor injects it. If one is lost (e.g., a transient KVM
>> failure), NAPI will not be scheduled and the driver stops consuming
>> buffers. The ring eventually fills up, the backend stops notifying, and
>> the queue is left permanently stuck.
>> 
>> Add a per-device watchdog that scans the RX queues once per second and
>> detects a queue that has a non-zero backlog while making no consumption
>> progress and receiving no new interrupt for rx_watchdog_timeo seconds
>> (default 5, 0 disables it). On detection, reschedule NAPI.
>> 
>> Signed-off-by: Longjun Tang <tanglongjun@kylinos.cn>
>> ---
>>  drivers/net/virtio_net.c | 104 +++++++++++++++++++++++++++++++++++++++
>>  1 file changed, 104 insertions(+)
>> 
>> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
>> index dd2c41ef6fbe..ec7c248080c5 100644
>> --- a/drivers/net/virtio_net.c
>> +++ b/drivers/net/virtio_net.c
>> @@ -36,6 +36,10 @@ module_param(csum, bool, 0444);
>>  module_param(gso, bool, 0444);
>>  module_param(napi_tx, bool, 0644);
>>  
>> +/* RX queue stall timeout in seconds; 0 disables the watchdog. */
>> +static unsigned int rx_watchdog_timeo = 5;
>> +module_param(rx_watchdog_timeo, uint, 0644);
>> +
>>  #define VIRTIO_OFFLOAD_MAP_MIN	46
>>  #define VIRTIO_OFFLOAD_MAP_MAX	47
>>  #define VIRTIO_FEATURES_MAP_MIN	65
>
>No module params pls. This is not a sane UAPI.

Dropped in v2; the timeout is now a fixed 5s constant.

>
>> @@ -318,6 +322,13 @@ struct send_queue {
>>  	dma_addr_t xsk_hdr_dma_addr;
>>  };
>>  
>> +struct virtnet_rq_watchdog {
>> +	u16 last_used;
>> +	u16 calls;
>> +	unsigned long jiffies;
>> +	bool stalled;
>> +};
>> +
>>  /* Internal representation of a receive virtqueue */
>>  struct receive_queue {
>>  	/* Virtqueue associated with this receive_queue */
>> @@ -332,6 +343,9 @@ struct receive_queue {
>>  	/* The number of rx notifications */
>>  	u16 calls;
>>  
>> +	/* RX watchdog state. */
>> +	struct virtnet_rq_watchdog watchdog;
>> +
>>  	/* Is dynamic interrupt moderation enabled? */
>>  	bool dim_enabled;
>>  
>> @@ -443,6 +457,9 @@ struct virtnet_info {
>>  	/* Work struct for setting rx mode */
>>  	struct work_struct rx_mode_work;
>>  
>> +	/* RX watchdog timer for stuck detection */
>> +	struct timer_list rx_watchdog;
>> +
>>  	/* Does the affinity hint is set for virtqueues? */
>>  	bool affinity_hint_set;
>>  
>> @@ -3067,6 +3084,80 @@ static int virtnet_poll(struct napi_struct *napi, int budget)
>>  	return received;
>>  }
>>  
>> +/*
>> + * An RX queue is stalled if it has a backlog but makes no progress and
>> + * gets no new interrupt for rx_watchdog_timeo seconds, usually because
>> + * an interrupt was lost.  Mark it stalled and reschedule NAPI to drain
>> + * the ring, refill the buffers.
>> + */
>> +static void virtnet_rx_watchdog(struct timer_list *t)
>> +{
>> +	struct virtnet_info *vi = timer_container_of(vi, t, rx_watchdog);
>> +	unsigned long timeout = rx_watchdog_timeo * HZ;
>> +	int i;
>> +
>> +	if (!rx_watchdog_timeo)
>> +		return;
>> +
>> +	for (i = 0; i < vi->curr_queue_pairs; i++) {
>> +		struct receive_queue *rq = &vi->rq[i];
>> +		u16 last_used = virtqueue_get_last_used_idx(rq->vq);
>> +		u16 calls = rq->calls;
>> +		bool backlog = virtqueue_poll(rq->vq, last_used);
>> +		bool progress = last_used != rq->watchdog.last_used;
>> +
>> +		if (!backlog || progress || calls != rq->watchdog.calls) {
>> +			rq->watchdog.last_used = last_used;
>> +			rq->watchdog.calls = calls;
>> +			rq->watchdog.jiffies = jiffies;
>> +			continue;
>> +		}
>> +
>> +		if (time_after(jiffies, rq->watchdog.jiffies + timeout)) {
>> +			unsigned int stall_ms =
>> +				jiffies_to_msecs(jiffies - rq->watchdog.jiffies);
>> +
>> +			netdev_warn(vi->dev,
>> +				    "RX queue %u stalled for %u ms, rescheduling NAPI\n",
>> +				    i, stall_ms);
>> +
>> +			rq->watchdog.stalled = true;
>> +			virtqueue_napi_schedule(&rq->napi, rq->vq);
>> +
>> +			/* Rate-limit to one event per timeout. */
>> +			rq->watchdog.jiffies = jiffies;
>> +		}
>> +	}
>> +
>> +	mod_timer(&vi->rx_watchdog, jiffies + HZ);
>> +}
>> +
>> +static void virtnet_rx_watchdog_init_queue(struct receive_queue *rq)
>> +{
>> +	rq->watchdog.last_used = virtqueue_get_last_used_idx(rq->vq);
>> +	rq->watchdog.calls = rq->calls;
>> +	rq->watchdog.jiffies = jiffies;
>> +	rq->watchdog.stalled = false;
>> +}
>> +
>> +static void virtnet_rx_watchdog_start(struct virtnet_info *vi)
>> +{
>> +	int i;
>> +
>> +	if (!rx_watchdog_timeo)
>> +		return;
>> +
>> +	for (i = 0; i < vi->curr_queue_pairs; i++)
>> +		virtnet_rx_watchdog_init_queue(&vi->rq[i]);
>> +
>> +	mod_timer(&vi->rx_watchdog, jiffies + HZ);
>> +}
>> +
>> +static void virtnet_rx_watchdog_stop(struct virtnet_info *vi)
>> +{
>> +	timer_delete_sync(&vi->rx_watchdog);
>> +}
>> +
>>  static void virtnet_disable_queue_pair(struct virtnet_info *vi, int qp_index)
>>  {
>>  	virtnet_napi_tx_disable(&vi->sq[qp_index]);
>> @@ -3230,6 +3321,8 @@ static int virtnet_open(struct net_device *dev)
>>  		netif_carrier_on(dev);
>>  	}
>>  
>> +	virtnet_rx_watchdog_start(vi);
>> +
>>  	return 0;
>>  
>>  err_enable_qp:
>> @@ -3759,6 +3852,7 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
>>  	struct virtio_net_rss_config_trailer old_rss_trailer;
>>  	struct net_device *dev = vi->dev;
>>  	struct scatterlist sg;
>> +	u16 old_queue_pairs = vi->curr_queue_pairs;
>>  
>>  	if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
>>  		return 0;
>> @@ -3816,7 +3910,14 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
>>  	if (vi->has_rss)
>>  		vi->rss_trailer.max_tx_vq = cpu_to_le16(queue_pairs);
>>  succ:
>> +	if (dev->flags & IFF_UP) {
>> +		/* Init new queues */
>> +		for (int i = old_queue_pairs; i < queue_pairs; i++)
>> +			virtnet_rx_watchdog_init_queue(&vi->rq[i]);
>> +	}
>> +
>>  	vi->curr_queue_pairs = queue_pairs;
>> +
>>  	if (dev->flags & IFF_UP) {
>>  		local_bh_disable();
>>  		for (int i = 0; i < vi->curr_queue_pairs; ++i)
>> @@ -3832,6 +3933,8 @@ static int virtnet_close(struct net_device *dev)
>>  	struct virtnet_info *vi = netdev_priv(dev);
>>  	int i;
>>  
>> +	virtnet_rx_watchdog_stop(vi);
>> +
>>  	/* Prevent the config change callback from changing carrier
>>  	 * after close
>>  	 */
>> @@ -6900,6 +7003,7 @@ static int virtnet_probe(struct virtio_device *vdev)
>>  	INIT_WORK(&vi->config_work, virtnet_config_changed_work);
>>  	INIT_WORK(&vi->rx_mode_work, virtnet_rx_mode_work);
>>  	disable_work(&vi->rx_mode_work);
>> +	timer_setup(&vi->rx_watchdog, virtnet_rx_watchdog, 0);
>>  
>>  	if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) {
>>  		vi->mergeable_rx_bufs = true;
>> -- 
>> 2.51.0

  reply	other threads:[~2026-11-20  7:15 UTC|newest]

Thread overview: 14+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-09  9:43 [PATCH 0/3] virtio_net: introduce an RX queue watchdog for stuck detection Longjun Tang
2026-11-09  9:43 ` [PATCH 1/3] virtio: add virtqueue_get_last_used_idx() helper Longjun Tang
2026-11-12 16:51   ` Jakub Kicinski
2026-11-20  7:10     ` Lange Tang
2026-11-12 22:50   ` Michael S. Tsirkin
2026-11-20  7:13     ` Lange Tang
2026-11-09  9:43 ` [PATCH 2/3] virtio_net: introduce an RX queue watchdog for stuck detection Longjun Tang
2026-11-12 16:53   ` Jakub Kicinski
2026-11-20  7:11     ` Lange Tang
2026-11-12 22:51   ` Michael S. Tsirkin
2026-11-20  7:14     ` Lange Tang [this message]
2026-11-09  9:43 ` [PATCH 3/3] virtio_net: add rx_timeouts per-queue statistic Longjun Tang
2026-11-12 22:52   ` Michael S. Tsirkin
2026-11-20  7:15     ` Lange Tang

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=7158076d.5a7a.1a1f7ce82d5.Coremail.lange_tang@163.com \
    --to=lange_tang@163.com \
    --cc=eperezma@redhat.com \
    --cc=jasowangio@gmail.com \
    --cc=kuba@kernel.org \
    --cc=mst@redhat.com \
    --cc=netdev@vger.kernel.org \
    --cc=tanglongjun@kylinos.cn \
    --cc=virtualization@lists.linux.dev \
    --cc=xuanzhuo@linux.alibaba.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).