[RFC PATCH 07/10] PM: hibernate: build one bio per contiguous run of the image

[Youngjun Park <youngjun.park@lge.com>]

The image is written with a bio per page, and the block layer merges
them back into requests.  A 5G image takes about 1.3M bio allocations
and nearly as many merges.

One page per bio dates from when the page lock was the completion
signal.  Commit 343df3c79c62 ("suspend: simplify block I/O handling")
removed that constraint, but the bio per page stayed.

Keep filling one bio while the next page follows the last one, and
submit it when the run breaks or the bio is full.  Size the bio from
the device limits.  Reads are unchanged.

The compressed path writes the image in rounds and waits for its
threads between them.  Submit the bio at the end of each round, so the
last pages of a round do not wait for the next one.

Without compression, a 5G image now takes about 12.7K bios, and writing
it on a VM took 12 to 15% less time.

Assisted-by: Claude:claude-fable-5
Assisted-by: Claude:claude-opus-5
Signed-off-by: Youngjun Park <youngjun.park@lge.com>
---
 kernel/power/swap.c | 96 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 88 insertions(+), 8 deletions(-)

diff --git a/kernel/power/swap.c b/kernel/power/swap.c
index 09c6c50bf837..34e47c662798 100644
--- a/kernel/power/swap.c
+++ b/kernel/power/swap.c
@@ -252,25 +252,55 @@ struct hib_bio_batch {
 	wait_queue_head_t	wait;
 	blk_status_t		error;
 	struct blk_plug		plug;
+	struct bio		*cur;		/* write bio still being filled */
+	unsigned int		max_pages;	/* what the device takes at once */
+	unsigned int		nr_pages;	/* what the next bio is sized for */
 };
 
+/* How much of the image the device is willing to take as one request. */
+static unsigned int hib_bio_max_pages(struct block_device *bdev)
+{
+	unsigned int pages = queue_max_bytes(bdev_get_queue(bdev)) >> PAGE_SHIFT;
+
+	pages = min_t(unsigned int, pages, bdev_max_segments(bdev));
+	return bio_max_segs(max(pages, 1U));
+}
+
 static void hib_init_batch(struct hib_bio_batch *hb)
 {
 	atomic_set(&hb->count, 0);
 	init_waitqueue_head(&hb->wait);
 	hb->error = BLK_STS_OK;
+	hb->cur = NULL;
+	hb->max_pages = hib_bio_max_pages(file_bdev(hib_resume_bdev_file));
+	hb->nr_pages = 1;
 	blk_start_plug(&hb->plug);
 }
 
+/* Hand over the bio being filled, if there is one. */
+static void hib_submit_cur(struct hib_bio_batch *hb)
+{
+	struct bio *bio = hb->cur;
+
+	if (!bio)
+		return;
+
+	hb->cur = NULL;
+	atomic_inc(&hb->count);
+	submit_bio(bio);
+}
+
 static void hib_finish_batch(struct hib_bio_batch *hb)
 {
+	hib_submit_cur(hb);
 	blk_finish_plug(&hb->plug);
 }
 
 static void hib_end_io(struct bio *bio)
 {
 	struct hib_bio_batch *hb = bio->bi_private;
-	struct page *page = bio_first_page_all(bio);
+	struct bvec_iter_all iter_all;
+	struct bio_vec *bv;
 
 	if (bio->bi_status) {
 		pr_alert("Read-error on swap-device (%u:%u:%Lu)\n",
@@ -278,11 +308,16 @@ static void hib_end_io(struct bio *bio)
 			 (unsigned long long)bio->bi_iter.bi_sector);
 	}
 
-	if (bio_data_dir(bio) == WRITE)
-		put_page(page);
-	else if (clean_pages_on_read)
-		flush_icache_range((unsigned long)page_address(page),
-				   (unsigned long)page_address(page) + PAGE_SIZE);
+	/* A write bio carries as many pages as the image was contiguous for. */
+	bio_for_each_segment_all(bv, bio, iter_all) {
+		struct page *page = bv->bv_page;
+
+		if (bio_data_dir(bio) == WRITE)
+			put_page(page);
+		else if (clean_pages_on_read)
+			flush_icache_range((unsigned long)page_address(page),
+					   (unsigned long)page_address(page) + PAGE_SIZE);
+	}
 
 	if (bio->bi_status && !hb->error)
 		hb->error = bio->bi_status;
@@ -298,17 +333,53 @@ static int hib_submit_io_sync(blk_opf_t opf, pgoff_t page_off, void *addr)
 			page_off * (PAGE_SIZE >> 9), addr, PAGE_SIZE, opf);
 }
 
+/*
+ * The image goes out a page at a time.  Keep filling one bio for as long as
+ * the next page lands right after the last, so that pages that are
+ * consecutive on the device reach it as one request rather than as many that
+ * the block layer then has to merge.
+ */
 static int hib_submit_io_async(blk_opf_t opf, pgoff_t page_off, void *addr,
 			 struct hib_bio_batch *hb)
 {
+	sector_t sector = page_off * (PAGE_SIZE >> 9);
+	bool write = op_is_write(opf);
 	struct bio *bio;
 
-	bio = bio_alloc(file_bdev(hib_resume_bdev_file), 1, opf,
+	if (write && hb->cur) {
+		bool contiguous = bio_end_sector(hb->cur) == sector;
+
+		if (contiguous && hb->cur->bi_vcnt < hb->cur->bi_max_vecs) {
+			bio_add_virt_nofail(hb->cur, addr, PAGE_SIZE);
+			return 0;
+		}
+
+		/*
+		 * Size the next bio for what the image has just shown.  A run
+		 * that carried on past this bio, or a bio that took more than
+		 * one page, says there are runs here worth the room.  Anything
+		 * else is an image in pieces, and a single page is all it can
+		 * use.
+		 */
+		hb->nr_pages = contiguous || hb->cur->bi_vcnt > 1 ?
+			       hb->max_pages : 1;
+
+		hib_submit_cur(hb);
+	}
+
+	bio = bio_alloc(file_bdev(hib_resume_bdev_file),
+			write ? hb->nr_pages : 1, opf,
 			GFP_NOIO | __GFP_HIGH);
-	bio->bi_iter.bi_sector = page_off * (PAGE_SIZE >> 9);
+	bio->bi_iter.bi_sector = sector;
 	bio_add_virt_nofail(bio, addr, PAGE_SIZE);
 	bio->bi_end_io = hib_end_io;
 	bio->bi_private = hb;
+
+	if (write) {
+		hb->cur = bio;
+		return 0;
+	}
+
 	atomic_inc(&hb->count);
 	submit_bio(bio);
 	return 0;
@@ -316,6 +387,9 @@ static int hib_submit_io_async(blk_opf_t opf, pgoff_t page_off, void *addr,
 
 static int hib_wait_io(struct hib_bio_batch *hb)
 {
+	/* Nothing will complete a bio that was never handed over. */
+	hib_submit_cur(hb);
+
 	/*
 	 * We are relying on the behavior of blk_plug that a thread with
 	 * a plug will flush the plug list before sleeping.
@@ -917,6 +991,12 @@ static int save_compressed_image(struct swap_map_handle *handle,
 			}
 		}
 
+		/*
+		 * Submit the rest of the round now.  Otherwise it can wait in
+		 * the bio until the next round fills it.
+		 */
+		hib_submit_cur(&hb);
+
 		wait_event(crc->done, atomic_read_acquire(&crc->stop));
 		atomic_set(&crc->stop, 0);
 	}
-- 
2.48.1