Re: NFS workload leaves nfsd threads in D state

[Christoph Hellwig <hch@lst.de>]

On Mon, Jul 10, 2023 at 05:40:42PM +0000, Chuck Lever III wrote:
> > blk_rq_init_flush(rq);
> > - rq->flush.seq |= REQ_FSEQ_POSTFLUSH;
> > + rq->flush.seq |= REQ_FSEQ_PREFLUSH;
> > spin_lock_irq(&fq->mq_flush_lock);
> > list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);
> > spin_unlock_irq(&fq->mq_flush_lock);
> 
> Thanks for the quick response. No change.

I'm a bit lost and still can't reprodce.  Below is a patch with the
only behavior differences I can find.  It has two "#if 1" blocks,
which I'll need to bisect to to find out which made it work (if any,
but I hope so).

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 5504719b970d59..67364e607f2d1d 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2927,6 +2927,7 @@ void blk_mq_submit_bio(struct bio *bio)
 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
 	struct blk_plug *plug = blk_mq_plug(bio);
 	const int is_sync = op_is_sync(bio->bi_opf);
+	bool is_flush = op_is_flush(bio->bi_opf);
 	struct blk_mq_hw_ctx *hctx;
 	struct request *rq;
 	unsigned int nr_segs = 1;
@@ -2967,16 +2968,23 @@ void blk_mq_submit_bio(struct bio *bio)
 		return;
 	}
 
-	if (op_is_flush(bio->bi_opf) && blk_insert_flush(rq))
-		return;
-
-	if (plug) {
-		blk_add_rq_to_plug(plug, rq);
-		return;
+#if 1	/* Variant 1, the plug is holding us back */
+	if (op_is_flush(bio->bi_opf)) {
+		if (blk_insert_flush(rq))
+			return;
+	} else {
+		if (plug) {
+			blk_add_rq_to_plug(plug, rq);
+			return;
+		}
 	}
+#endif
 
 	hctx = rq->mq_hctx;
 	if ((rq->rq_flags & RQF_USE_SCHED) ||
+#if 1	/* Variant 2 (unlikely), blk_mq_try_issue_directly causes problems */
+	    is_flush || 
+#endif
 	    (hctx->dispatch_busy && (q->nr_hw_queues == 1 || !is_sync))) {
 		blk_mq_insert_request(rq, 0);
 		blk_mq_run_hw_queue(hctx, true);