From: "Lange Tang" <lange_tang@163.com>
To: "mst@redhat.com" <mst@redhat.com>
Cc: "Jakub Kicinski" <kuba@kernel.org>,
"Eugenio Perez" <eperezma@redhat.com>,
"jasowangio@gmail.com" <jasowangio@gmail.com>,
"xuanzhuo@linux.alibaba.com" <xuanzhuo@linux.alibaba.com>,
"virtualization@lists.linux.dev" <virtualization@lists.linux.dev>,
"netdev@vger.kernel.org" <netdev@vger.kernel.org>,
"Tang Longjun" <tanglongjun@kylinos.cn>
Subject: Re:Re: [PATCH 2/3] virtio_net: introduce an RX queue watchdog for stuck detection
Date: Fri, 20 Nov 2026 15:14:28 +0800 (CST) [thread overview]
Message-ID: <7158076d.5a7a.1a1f7ce82d5.Coremail.lange_tang@163.com> (raw)
In-Reply-To: <20261112175123-mutt-send-email-mst@kernel.org>
At 2026-11-13 06:51:46, "Michael S. Tsirkin" <mst@redhat.com> wrote:
>On Mon, Nov 09, 2026 at 05:43:09PM +0800, Longjun Tang wrote:
>> From: Longjun Tang <tanglongjun@kylinos.cn>
>>
>> The backend raises an interrupt after adding buffers to the used ring,
>> and the hypervisor injects it. If one is lost (e.g., a transient KVM
>> failure), NAPI will not be scheduled and the driver stops consuming
>> buffers. The ring eventually fills up, the backend stops notifying, and
>> the queue is left permanently stuck.
>>
>> Add a per-device watchdog that scans the RX queues once per second and
>> detects a queue that has a non-zero backlog while making no consumption
>> progress and receiving no new interrupt for rx_watchdog_timeo seconds
>> (default 5, 0 disables it). On detection, reschedule NAPI.
>>
>> Signed-off-by: Longjun Tang <tanglongjun@kylinos.cn>
>> ---
>> drivers/net/virtio_net.c | 104 +++++++++++++++++++++++++++++++++++++++
>> 1 file changed, 104 insertions(+)
>>
>> diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
>> index dd2c41ef6fbe..ec7c248080c5 100644
>> --- a/drivers/net/virtio_net.c
>> +++ b/drivers/net/virtio_net.c
>> @@ -36,6 +36,10 @@ module_param(csum, bool, 0444);
>> module_param(gso, bool, 0444);
>> module_param(napi_tx, bool, 0644);
>>
>> +/* RX queue stall timeout in seconds; 0 disables the watchdog. */
>> +static unsigned int rx_watchdog_timeo = 5;
>> +module_param(rx_watchdog_timeo, uint, 0644);
>> +
>> #define VIRTIO_OFFLOAD_MAP_MIN 46
>> #define VIRTIO_OFFLOAD_MAP_MAX 47
>> #define VIRTIO_FEATURES_MAP_MIN 65
>
>No module params pls. This is not a sane UAPI.
Dropped in v2; the timeout is now a fixed 5s constant.
>
>> @@ -318,6 +322,13 @@ struct send_queue {
>> dma_addr_t xsk_hdr_dma_addr;
>> };
>>
>> +struct virtnet_rq_watchdog {
>> + u16 last_used;
>> + u16 calls;
>> + unsigned long jiffies;
>> + bool stalled;
>> +};
>> +
>> /* Internal representation of a receive virtqueue */
>> struct receive_queue {
>> /* Virtqueue associated with this receive_queue */
>> @@ -332,6 +343,9 @@ struct receive_queue {
>> /* The number of rx notifications */
>> u16 calls;
>>
>> + /* RX watchdog state. */
>> + struct virtnet_rq_watchdog watchdog;
>> +
>> /* Is dynamic interrupt moderation enabled? */
>> bool dim_enabled;
>>
>> @@ -443,6 +457,9 @@ struct virtnet_info {
>> /* Work struct for setting rx mode */
>> struct work_struct rx_mode_work;
>>
>> + /* RX watchdog timer for stuck detection */
>> + struct timer_list rx_watchdog;
>> +
>> /* Does the affinity hint is set for virtqueues? */
>> bool affinity_hint_set;
>>
>> @@ -3067,6 +3084,80 @@ static int virtnet_poll(struct napi_struct *napi, int budget)
>> return received;
>> }
>>
>> +/*
>> + * An RX queue is stalled if it has a backlog but makes no progress and
>> + * gets no new interrupt for rx_watchdog_timeo seconds, usually because
>> + * an interrupt was lost. Mark it stalled and reschedule NAPI to drain
>> + * the ring, refill the buffers.
>> + */
>> +static void virtnet_rx_watchdog(struct timer_list *t)
>> +{
>> + struct virtnet_info *vi = timer_container_of(vi, t, rx_watchdog);
>> + unsigned long timeout = rx_watchdog_timeo * HZ;
>> + int i;
>> +
>> + if (!rx_watchdog_timeo)
>> + return;
>> +
>> + for (i = 0; i < vi->curr_queue_pairs; i++) {
>> + struct receive_queue *rq = &vi->rq[i];
>> + u16 last_used = virtqueue_get_last_used_idx(rq->vq);
>> + u16 calls = rq->calls;
>> + bool backlog = virtqueue_poll(rq->vq, last_used);
>> + bool progress = last_used != rq->watchdog.last_used;
>> +
>> + if (!backlog || progress || calls != rq->watchdog.calls) {
>> + rq->watchdog.last_used = last_used;
>> + rq->watchdog.calls = calls;
>> + rq->watchdog.jiffies = jiffies;
>> + continue;
>> + }
>> +
>> + if (time_after(jiffies, rq->watchdog.jiffies + timeout)) {
>> + unsigned int stall_ms =
>> + jiffies_to_msecs(jiffies - rq->watchdog.jiffies);
>> +
>> + netdev_warn(vi->dev,
>> + "RX queue %u stalled for %u ms, rescheduling NAPI\n",
>> + i, stall_ms);
>> +
>> + rq->watchdog.stalled = true;
>> + virtqueue_napi_schedule(&rq->napi, rq->vq);
>> +
>> + /* Rate-limit to one event per timeout. */
>> + rq->watchdog.jiffies = jiffies;
>> + }
>> + }
>> +
>> + mod_timer(&vi->rx_watchdog, jiffies + HZ);
>> +}
>> +
>> +static void virtnet_rx_watchdog_init_queue(struct receive_queue *rq)
>> +{
>> + rq->watchdog.last_used = virtqueue_get_last_used_idx(rq->vq);
>> + rq->watchdog.calls = rq->calls;
>> + rq->watchdog.jiffies = jiffies;
>> + rq->watchdog.stalled = false;
>> +}
>> +
>> +static void virtnet_rx_watchdog_start(struct virtnet_info *vi)
>> +{
>> + int i;
>> +
>> + if (!rx_watchdog_timeo)
>> + return;
>> +
>> + for (i = 0; i < vi->curr_queue_pairs; i++)
>> + virtnet_rx_watchdog_init_queue(&vi->rq[i]);
>> +
>> + mod_timer(&vi->rx_watchdog, jiffies + HZ);
>> +}
>> +
>> +static void virtnet_rx_watchdog_stop(struct virtnet_info *vi)
>> +{
>> + timer_delete_sync(&vi->rx_watchdog);
>> +}
>> +
>> static void virtnet_disable_queue_pair(struct virtnet_info *vi, int qp_index)
>> {
>> virtnet_napi_tx_disable(&vi->sq[qp_index]);
>> @@ -3230,6 +3321,8 @@ static int virtnet_open(struct net_device *dev)
>> netif_carrier_on(dev);
>> }
>>
>> + virtnet_rx_watchdog_start(vi);
>> +
>> return 0;
>>
>> err_enable_qp:
>> @@ -3759,6 +3852,7 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
>> struct virtio_net_rss_config_trailer old_rss_trailer;
>> struct net_device *dev = vi->dev;
>> struct scatterlist sg;
>> + u16 old_queue_pairs = vi->curr_queue_pairs;
>>
>> if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ))
>> return 0;
>> @@ -3816,7 +3910,14 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs)
>> if (vi->has_rss)
>> vi->rss_trailer.max_tx_vq = cpu_to_le16(queue_pairs);
>> succ:
>> + if (dev->flags & IFF_UP) {
>> + /* Init new queues */
>> + for (int i = old_queue_pairs; i < queue_pairs; i++)
>> + virtnet_rx_watchdog_init_queue(&vi->rq[i]);
>> + }
>> +
>> vi->curr_queue_pairs = queue_pairs;
>> +
>> if (dev->flags & IFF_UP) {
>> local_bh_disable();
>> for (int i = 0; i < vi->curr_queue_pairs; ++i)
>> @@ -3832,6 +3933,8 @@ static int virtnet_close(struct net_device *dev)
>> struct virtnet_info *vi = netdev_priv(dev);
>> int i;
>>
>> + virtnet_rx_watchdog_stop(vi);
>> +
>> /* Prevent the config change callback from changing carrier
>> * after close
>> */
>> @@ -6900,6 +7003,7 @@ static int virtnet_probe(struct virtio_device *vdev)
>> INIT_WORK(&vi->config_work, virtnet_config_changed_work);
>> INIT_WORK(&vi->rx_mode_work, virtnet_rx_mode_work);
>> disable_work(&vi->rx_mode_work);
>> + timer_setup(&vi->rx_watchdog, virtnet_rx_watchdog, 0);
>>
>> if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) {
>> vi->mergeable_rx_bufs = true;
>> --
>> 2.51.0
next prev parent reply other threads:[~2026-11-20 7:15 UTC|newest]
Thread overview: 14+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-09 9:43 [PATCH 0/3] virtio_net: introduce an RX queue watchdog for stuck detection Longjun Tang
2026-11-09 9:43 ` [PATCH 1/3] virtio: add virtqueue_get_last_used_idx() helper Longjun Tang
2026-11-12 16:51 ` Jakub Kicinski
2026-11-20 7:10 ` Lange Tang
2026-11-12 22:50 ` Michael S. Tsirkin
2026-11-20 7:13 ` Lange Tang
2026-11-09 9:43 ` [PATCH 2/3] virtio_net: introduce an RX queue watchdog for stuck detection Longjun Tang
2026-11-12 16:53 ` Jakub Kicinski
2026-11-20 7:11 ` Lange Tang
2026-11-12 22:51 ` Michael S. Tsirkin
2026-11-20 7:14 ` Lange Tang [this message]
2026-11-09 9:43 ` [PATCH 3/3] virtio_net: add rx_timeouts per-queue statistic Longjun Tang
2026-11-12 22:52 ` Michael S. Tsirkin
2026-11-20 7:15 ` Lange Tang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=7158076d.5a7a.1a1f7ce82d5.Coremail.lange_tang@163.com \
--to=lange_tang@163.com \
--cc=eperezma@redhat.com \
--cc=jasowangio@gmail.com \
--cc=kuba@kernel.org \
--cc=mst@redhat.com \
--cc=netdev@vger.kernel.org \
--cc=tanglongjun@kylinos.cn \
--cc=virtualization@lists.linux.dev \
--cc=xuanzhuo@linux.alibaba.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).