[PATCH] DIO rewrite

[PATCH] DIO rewrite

[Kent Overstreet]

This is the dio rewrite based on immutable biovecs that I've been working on
off and on for quite some time now; I think it's getting close to ready.

It passes xfstests with xfs and CONFIG_XFS_DEBUG=y, though it definitely needs
more review and outside testing. It doesn't quite work with btrfs but I think
with a bit of help from the btrfs people that one will be easy to figure out.

This  this cuts fs/direct-io.c almost in half by lines of code, makes it
_vastly_ easier to understand, and roughly halves the cpu overhead of the dio
code in the profiles I've looked at. It's only stage one of my master plan,
though.

Eventually, the goal is to get the filesystem direct_IO() method to take bios
(at which point it'll be analagous to a make_request fn, probably) - that'll
have a lot of benefits I won't go into here but it's going to take some
delicate surgery to get here. This code starts to restructure things in that
direction, doing about as much as I could without going outside of
fs/direct-io.c.

The patch series is on top of Jens' for-3.14/core branch, and includes all the
prereqs that haven't gone in yet. It's available in my git repo:

git://evilpiepirate.org/~kent/linux-bcache.git dio

[PATCH 01/11] block: Make generic_make_request handle arbitrary sized bios

[Kent Overstreet]

The way the block layer is currently written, it goes to great lengths
to avoid having to split bios; upper layer code (such as bio_add_page())
checks what the underlying device can handle and tries to always create
bios that don't need to be split.

But this approach becomes unwieldy and eventually breaks down with
stacked devices and devices with dynamic limits, and it adds a lot of
complexity. If the block layer could split bios as needed, we could
eliminate a lot of complexity elsewhere - particularly in stacked
drivers. Code that creates bios can then create whatever size bios are
convenient, and more importantly stacked drivers don't have to deal with
both their own bio size limitations and the limitations of the
(potentially multiple) devices underneath them.  In the future this will
let us delete merge_bvec_fn and a bunch of other code.

We do this by adding calls to blk_queue_split() to the various
make_request functions that need it - a few can already handle arbitrary
size bios. Note that we add the call _after_ any call to blk_queue_bounce();
this means that blk_queue_split() and blk_recalc_rq_segments() don't need to be
concerned with bouncing affecting segment merging.

Some make_request_fns were simple enough to audit and verify they don't
need blk_queue_split() calls. The skipped ones are:

 * nfhd_make_request (arch/m68k/emu/nfblock.c)
 * axon_ram_make_request (arch/powerpc/sysdev/axonram.c)
 * simdisk_make_request (arch/xtensa/platforms/iss/simdisk.c)
 * brd_make_request (ramdisk - drivers/block/brd.c)
 * loop_make_request
 * null_queue_bio
 * bcache's make_request fns

Some others are almost certainly safe to remove now, but will be left for future
patches.

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Neil Brown <neilb@suse.de>
Cc: Alasdair Kergon <agk@redhat.com>
Cc: dm-devel@redhat.com
---
 block/blk-core.c                            |  19 ++--
 block/blk-merge.c                           | 150 ++++++++++++++++++++++++++--
 block/blk-mq.c                              |   2 +
 drivers/block/drbd/drbd_req.c               |   2 +
 drivers/block/mtip32xx/mtip32xx.c           |   6 +-
 drivers/block/nvme-core.c                   |   2 +
 drivers/block/pktcdvd.c                     |   6 +-
 drivers/block/ps3vram.c                     |   2 +
 drivers/block/rsxx/dev.c                    |   2 +
 drivers/block/umem.c                        |   2 +
 drivers/md/dm.c                             |   2 +
 drivers/md/md.c                             |   2 +
 drivers/s390/block/dcssblk.c                |   2 +
 drivers/s390/block/xpram.c                  |   2 +
 drivers/staging/lustre/lustre/llite/lloop.c |   2 +
 drivers/staging/zram/zram_drv.c             |   2 +
 include/linux/blkdev.h                      |   3 +
 17 files changed, 185 insertions(+), 23 deletions(-)

diff --git a/block/blk-core.c b/block/blk-core.c
index 5da8e90..5cc7171 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -566,6 +566,10 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
 	if (q->id < 0)
 		goto fail_c;
 
+	q->bio_split = bioset_create(4, 0);
+	if (!q->bio_split)
+		goto fail_id;
+
 	q->backing_dev_info.ra_pages =
 			(VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
 	q->backing_dev_info.state = 0;
@@ -575,7 +579,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
 
 	err = bdi_init(&q->backing_dev_info);
 	if (err)
-		goto fail_id;
+		goto fail_split;
 
 	setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
 		    laptop_mode_timer_fn, (unsigned long) q);
@@ -620,6 +624,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
 
 fail_bdi:
 	bdi_destroy(&q->backing_dev_info);
+fail_split:
+	bioset_free(q->bio_split);
 fail_id:
 	ida_simple_remove(&blk_queue_ida, q->id);
 fail_c:
@@ -1472,6 +1478,8 @@ void blk_queue_bio(struct request_queue *q, struct bio *bio)
 	struct request *req;
 	unsigned int request_count = 0;
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	/*
 	 * low level driver can indicate that it wants pages above a
 	 * certain limit bounced to low memory (ie for highmem, or even
@@ -1694,15 +1702,6 @@ generic_make_request_checks(struct bio *bio)
 		goto end_io;
 	}
 
-	if (likely(bio_is_rw(bio) &&
-		   nr_sectors > queue_max_hw_sectors(q))) {
-		printk(KERN_ERR "bio too big device %s (%u > %u)\n",
-		       bdevname(bio->bi_bdev, b),
-		       bio_sectors(bio),
-		       queue_max_hw_sectors(q));
-		goto end_io;
-	}
-
 	part = bio->bi_bdev->bd_part;
 	if (should_fail_request(part, bio->bi_iter.bi_size) ||
 	    should_fail_request(&part_to_disk(part)->part0,
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 8f8adaa..c1403ee 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -9,11 +9,149 @@
 
 #include "blk.h"
 
+static struct bio *blk_bio_discard_split(struct request_queue *q,
+					 struct bio *bio,
+					 struct bio_set *bs)
+{
+	unsigned int max_discard_sectors, granularity;
+	int alignment;
+	sector_t tmp;
+	unsigned split_sectors;
+
+	/* Zero-sector (unknown) and one-sector granularities are the same.  */
+	granularity = max(q->limits.discard_granularity >> 9, 1U);
+
+	max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9);
+	max_discard_sectors -= max_discard_sectors % granularity;
+
+	if (unlikely(!max_discard_sectors)) {
+		/* XXX: warn */
+		return NULL;
+	}
+
+	if (bio_sectors(bio) <= max_discard_sectors)
+		return NULL;
+
+	split_sectors = max_discard_sectors;
+
+	/*
+	 * If the next starting sector would be misaligned, stop the discard at
+	 * the previous aligned sector.
+	 */
+	alignment = (q->limits.discard_alignment >> 9) % granularity;
+
+	tmp = bio->bi_iter.bi_sector + split_sectors - alignment;
+	tmp = sector_div(tmp, granularity);
+
+	if (split_sectors > tmp)
+		split_sectors -= tmp;
+
+	return bio_split(bio, split_sectors, GFP_NOIO, bs);
+}
+
+static struct bio *blk_bio_write_same_split(struct request_queue *q,
+					    struct bio *bio,
+					    struct bio_set *bs)
+{
+	if (!q->limits.max_write_same_sectors)
+		return NULL;
+
+	if (bio_sectors(bio) <= q->limits.max_write_same_sectors)
+		return NULL;
+
+	return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs);
+}
+
+static struct bio *blk_bio_segment_split(struct request_queue *q,
+					 struct bio *bio,
+					 struct bio_set *bs)
+{
+	struct bio *split;
+	struct bio_vec bv, bvprv;
+	struct bvec_iter iter;
+	unsigned seg_size = 0, nsegs = 0;
+	int prev = 0;
+
+	struct bvec_merge_data bvm = {
+		.bi_bdev	= bio->bi_bdev,
+		.bi_sector	= bio->bi_iter.bi_sector,
+		.bi_size	= 0,
+		.bi_rw		= bio->bi_rw,
+	};
+
+	bio_for_each_segment(bv, bio, iter) {
+		if (q->merge_bvec_fn &&
+		    q->merge_bvec_fn(q, &bvm, &bv) < (int) bv.bv_len)
+			goto split;
+
+		bvm.bi_size += bv.bv_len;
+
+		if (bvm.bi_size >> 9 > queue_max_sectors(q))
+			goto split;
+
+		if (prev && blk_queue_cluster(q)) {
+			if (seg_size + bv.bv_len > queue_max_segment_size(q))
+				goto new_segment;
+			if (!BIOVEC_PHYS_MERGEABLE(&bvprv, &bv))
+				goto new_segment;
+			if (!BIOVEC_SEG_BOUNDARY(q, &bvprv, &bv))
+				goto new_segment;
+
+			seg_size += bv.bv_len;
+			bvprv = bv;
+			prev = 1;
+			continue;
+		}
+new_segment:
+		if (nsegs == queue_max_segments(q))
+			goto split;
+
+		nsegs++;
+		bvprv = bv;
+		prev = 1;
+		seg_size = bv.bv_len;
+	}
+
+	return NULL;
+split:
+	split = bio_clone_bioset(bio, GFP_NOIO, bs);
+
+	split->bi_iter.bi_size -= iter.bi_size;
+	bio->bi_iter = iter;
+
+	if (bio_integrity(bio)) {
+		bio_integrity_advance(bio, split->bi_iter.bi_size);
+		bio_integrity_trim(split, 0, bio_sectors(split));
+	}
+
+	return split;
+}
+
+void blk_queue_split(struct request_queue *q, struct bio **bio,
+		     struct bio_set *bs)
+{
+	struct bio *split;
+
+	if ((*bio)->bi_rw & REQ_DISCARD)
+		split = blk_bio_discard_split(q, *bio, bs);
+	else if ((*bio)->bi_rw & REQ_WRITE_SAME)
+		split = blk_bio_write_same_split(q, *bio, bs);
+	else
+		split = blk_bio_segment_split(q, *bio, q->bio_split);
+
+	if (split) {
+		bio_chain(split, *bio);
+		generic_make_request(*bio);
+		*bio = split;
+	}
+}
+EXPORT_SYMBOL(blk_queue_split);
+
 static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
 					     struct bio *bio)
 {
 	struct bio_vec bv, bvprv = { NULL };
-	int cluster, high, highprv = 1;
+	int cluster, prev = 0;
 	unsigned int seg_size, nr_phys_segs;
 	struct bio *fbio, *bbio;
 	struct bvec_iter iter;
@@ -27,13 +165,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
 	nr_phys_segs = 0;
 	for_each_bio(bio) {
 		bio_for_each_segment(bv, bio, iter) {
-			/*
-			 * the trick here is making sure that a high page is
-			 * never considered part of another segment, since that
-			 * might change with the bounce page.
-			 */
-			high = page_to_pfn(bv.bv_page) > queue_bounce_pfn(q);
-			if (!high && !highprv && cluster) {
+			if (prev && cluster) {
 				if (seg_size + bv.bv_len
 				    > queue_max_segment_size(q))
 					goto new_segment;
@@ -53,8 +185,8 @@ new_segment:
 
 			nr_phys_segs++;
 			bvprv = bv;
+			prev = 1;
 			seg_size = bv.bv_len;
-			highprv = high;
 		}
 		bbio = bio;
 	}
diff --git a/block/blk-mq.c b/block/blk-mq.c
index e4fbcc3..d29be09 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -912,6 +912,8 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
 
 	blk_queue_bounce(q, &bio);
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	if (use_plug && blk_attempt_plug_merge(q, bio, &request_count))
 		return;
 
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index 104a040..941a69c 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -1275,6 +1275,8 @@ void drbd_make_request(struct request_queue *q, struct bio *bio)
 	struct drbd_conf *mdev = (struct drbd_conf *) q->queuedata;
 	unsigned long start_time;
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	start_time = jiffies;
 
 	/*
diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c
index 52b2f2a..3ff8120 100644
--- a/drivers/block/mtip32xx/mtip32xx.c
+++ b/drivers/block/mtip32xx/mtip32xx.c
@@ -3967,6 +3967,10 @@ static void mtip_make_request(struct request_queue *queue, struct bio *bio)
 	int nents = 0;
 	int tag = 0, unaligned = 0;
 
+	blk_queue_bounce(queue, &bio);
+
+	blk_queue_split(queue, &bio, queue->bio_split);
+
 	if (unlikely(dd->dd_flag & MTIP_DDF_STOP_IO)) {
 		if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
 							&dd->dd_flag))) {
@@ -4016,8 +4020,6 @@ static void mtip_make_request(struct request_queue *queue, struct bio *bio)
 
 	sg = mtip_hw_get_scatterlist(dd, &tag, unaligned);
 	if (likely(sg != NULL)) {
-		blk_queue_bounce(queue, &bio);
-
 		if (unlikely((bio)->bi_vcnt > MTIP_MAX_SG)) {
 			dev_warn(&dd->pdev->dev,
 				"Maximum number of SGL entries exceeded\n");
diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c
index 1f14ac4..cfb8f7b 100644
--- a/drivers/block/nvme-core.c
+++ b/drivers/block/nvme-core.c
@@ -707,6 +707,8 @@ static void nvme_make_request(struct request_queue *q, struct bio *bio)
 	struct nvme_queue *nvmeq = get_nvmeq(ns->dev);
 	int result = -EBUSY;
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	if (!nvmeq) {
 		put_nvmeq(NULL);
 		bio_endio(bio, -EIO);
diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index 3dda09a..38799f8 100644
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -2442,6 +2442,10 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)
 	char b[BDEVNAME_SIZE];
 	struct bio *split;
 
+	blk_queue_bounce(q, &bio);
+
+	blk_queue_split(q, &bio, q->bio_split);
+
 	pd = q->queuedata;
 	if (!pd) {
 		pr_err("%s incorrect request queue\n",
@@ -2472,8 +2476,6 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio)
 		goto end_io;
 	}
 
-	blk_queue_bounce(q, &bio);
-
 	do {
 		sector_t zone = get_zone(bio->bi_iter.bi_sector, pd);
 		sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd);
diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c
index ef45cfb..a995972 100644
--- a/drivers/block/ps3vram.c
+++ b/drivers/block/ps3vram.c
@@ -603,6 +603,8 @@ static void ps3vram_make_request(struct request_queue *q, struct bio *bio)
 	struct ps3vram_priv *priv = ps3_system_bus_get_drvdata(dev);
 	int busy;
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	dev_dbg(&dev->core, "%s\n", __func__);
 
 	spin_lock_irq(&priv->lock);
diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c
index 2839d37..ff074a3 100644
--- a/drivers/block/rsxx/dev.c
+++ b/drivers/block/rsxx/dev.c
@@ -169,6 +169,8 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio)
 	struct rsxx_bio_meta *bio_meta;
 	int st = -EINVAL;
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	might_sleep();
 
 	if (!card)
diff --git a/drivers/block/umem.c b/drivers/block/umem.c
index 4cf81b5..13d577c 100644
--- a/drivers/block/umem.c
+++ b/drivers/block/umem.c
@@ -531,6 +531,8 @@ static void mm_make_request(struct request_queue *q, struct bio *bio)
 		 (unsigned long long)bio->bi_iter.bi_sector,
 		 bio->bi_iter.bi_size);
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	spin_lock_irq(&card->lock);
 	*card->biotail = bio;
 	bio->bi_next = NULL;
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 44a2fa6..0bfb050 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -1500,6 +1500,8 @@ static void dm_request(struct request_queue *q, struct bio *bio)
 {
 	struct mapped_device *md = q->queuedata;
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	if (dm_request_based(md))
 		blk_queue_bio(q, bio);
 	else
diff --git a/drivers/md/md.c b/drivers/md/md.c
index b07fed3..de39a8a 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -256,6 +256,8 @@ static void md_make_request(struct request_queue *q, struct bio *bio)
 	int cpu;
 	unsigned int sectors;
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	if (mddev == NULL || mddev->pers == NULL
 	    || !mddev->ready) {
 		bio_io_error(bio);
diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c
index ebf41e2..db33cd3 100644
--- a/drivers/s390/block/dcssblk.c
+++ b/drivers/s390/block/dcssblk.c
@@ -815,6 +815,8 @@ dcssblk_make_request(struct request_queue *q, struct bio *bio)
 	unsigned long source_addr;
 	unsigned long bytes_done;
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	bytes_done = 0;
 	dev_info = bio->bi_bdev->bd_disk->private_data;
 	if (dev_info == NULL)
diff --git a/drivers/s390/block/xpram.c b/drivers/s390/block/xpram.c
index 3e530f9..3ae29a0 100644
--- a/drivers/s390/block/xpram.c
+++ b/drivers/s390/block/xpram.c
@@ -190,6 +190,8 @@ static void xpram_make_request(struct request_queue *q, struct bio *bio)
 	unsigned long page_addr;
 	unsigned long bytes;
 
+	blk_queue_split(q, &bio, q->bio_split);
+
 	if ((bio->bi_iter.bi_sector & 7) != 0 ||
 	    (bio->bi_iter.bi_size & 4095) != 0)
 		/* Request is not page-aligned. */
diff --git a/drivers/staging/lustre/lustre/llite/lloop.c b/drivers/staging/lustre/lustre/llite/lloop.c
index 581ff78..776d44c 100644
--- a/drivers/staging/lustre/lustre/llite/lloop.c
+++ b/drivers/staging/lustre/lustre/llite/lloop.c
@@ -344,6 +344,8 @@ static void loop_make_request(struct request_queue *q, struct bio *old_bio)
 	int rw = bio_rw(old_bio);
 	int inactive;
 
+	blk_queue_split(q, &old_bio, q->bio_split);
+
 	if (!lo)
 		goto err;
 
diff --git a/drivers/staging/zram/zram_drv.c b/drivers/staging/zram/zram_drv.c
index 6f98838..56eb09e 100644
--- a/drivers/staging/zram/zram_drv.c
+++ b/drivers/staging/zram/zram_drv.c
@@ -734,6 +734,8 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio)
 {
 	struct zram *zram = queue->queuedata;
 
+	blk_queue_split(queue, &bio, queue->bio_split);
+
 	down_read(&zram->init_lock);
 	if (unlikely(!zram->init_done))
 		goto error;
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 02cb6f0..88d4e7b 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -476,6 +476,7 @@ struct request_queue {
 	wait_queue_head_t	mq_freeze_wq;
 	struct percpu_counter	mq_usage_counter;
 	struct list_head	all_q_node;
+	struct bio_set		*bio_split;
 };
 
 #define QUEUE_FLAG_QUEUED	1	/* uses generic tag queueing */
@@ -787,6 +788,8 @@ extern void blk_rq_unprep_clone(struct request *rq);
 extern int blk_insert_cloned_request(struct request_queue *q,
 				     struct request *rq);
 extern void blk_delay_queue(struct request_queue *, unsigned long);
+extern void blk_queue_split(struct request_queue *, struct bio **,
+			    struct bio_set *);
 extern void blk_recount_segments(struct request_queue *, struct bio *);
 extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
 extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
-- 
1.8.4.4

[PATCH 02/11] block: Gut bio_add_page()

[Kent Overstreet]

Since generic_make_request() can now handle arbitrary size bios, all we
have to do is make sure the bvec array doesn't overflow.

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
Cc: Jens Axboe <axboe@kernel.dk>
---
 fs/bio.c | 137 ++++++++++++++++++++++++++-------------------------------------
 1 file changed, 57 insertions(+), 80 deletions(-)

diff --git a/fs/bio.c b/fs/bio.c
index 75c49a3..da27557 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -690,9 +690,23 @@ int bio_get_nr_vecs(struct block_device *bdev)
 }
 EXPORT_SYMBOL(bio_get_nr_vecs);
 
-static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
-			  *page, unsigned int len, unsigned int offset,
-			  unsigned int max_sectors)
+/**
+ *	bio_add_pc_page	-	attempt to add page to bio
+ *	@q: the target queue
+ *	@bio: destination bio
+ *	@page: page to add
+ *	@len: vec entry length
+ *	@offset: vec entry offset
+ *
+ *	Attempt to add a page to the bio_vec maplist. This can fail for a
+ *	number of reasons, such as the bio being full or target block device
+ *	limitations. The target block device must allow bio's up to PAGE_SIZE,
+ *	so it is always possible to add a single page to an empty bio.
+ *
+ *	This should only be used by REQ_PC bios.
+ */
+int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page
+		    *page, unsigned int len, unsigned int offset)
 {
 	int retried_segments = 0;
 	struct bio_vec *bvec;
@@ -703,7 +717,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
 	if (unlikely(bio_flagged(bio, BIO_CLONED)))
 		return 0;
 
-	if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors)
+	if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q))
 		return 0;
 
 	/*
@@ -716,28 +730,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
 
 		if (page == prev->bv_page &&
 		    offset == prev->bv_offset + prev->bv_len) {
-			unsigned int prev_bv_len = prev->bv_len;
 			prev->bv_len += len;
-
-			if (q->merge_bvec_fn) {
-				struct bvec_merge_data bvm = {
-					/* prev_bvec is already charged in
-					   bi_size, discharge it in order to
-					   simulate merging updated prev_bvec
-					   as new bvec. */
-					.bi_bdev = bio->bi_bdev,
-					.bi_sector = bio->bi_iter.bi_sector,
-					.bi_size = bio->bi_iter.bi_size -
-						prev_bv_len,
-					.bi_rw = bio->bi_rw,
-				};
-
-				if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len) {
-					prev->bv_len -= len;
-					return 0;
-				}
-			}
-
 			goto done;
 		}
 	}
@@ -768,31 +761,6 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
 	bvec->bv_len = len;
 	bvec->bv_offset = offset;
 
-	/*
-	 * if queue has other restrictions (eg varying max sector size
-	 * depending on offset), it can specify a merge_bvec_fn in the
-	 * queue to get further control
-	 */
-	if (q->merge_bvec_fn) {
-		struct bvec_merge_data bvm = {
-			.bi_bdev = bio->bi_bdev,
-			.bi_sector = bio->bi_iter.bi_sector,
-			.bi_size = bio->bi_iter.bi_size,
-			.bi_rw = bio->bi_rw,
-		};
-
-		/*
-		 * merge_bvec_fn() returns number of bytes it can accept
-		 * at this offset
-		 */
-		if (q->merge_bvec_fn(q, &bvm, bvec) < bvec->bv_len) {
-			bvec->bv_page = NULL;
-			bvec->bv_len = 0;
-			bvec->bv_offset = 0;
-			return 0;
-		}
-	}
-
 	/* If we may be able to merge these biovecs, force a recount */
 	if (bio->bi_vcnt && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
 		bio->bi_flags &= ~(1 << BIO_SEG_VALID);
@@ -803,28 +771,6 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
 	bio->bi_iter.bi_size += len;
 	return len;
 }
-
-/**
- *	bio_add_pc_page	-	attempt to add page to bio
- *	@q: the target queue
- *	@bio: destination bio
- *	@page: page to add
- *	@len: vec entry length
- *	@offset: vec entry offset
- *
- *	Attempt to add a page to the bio_vec maplist. This can fail for a
- *	number of reasons, such as the bio being full or target block device
- *	limitations. The target block device must allow bio's up to PAGE_SIZE,
- *	so it is always possible to add a single page to an empty bio.
- *
- *	This should only be used by REQ_PC bios.
- */
-int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page,
-		    unsigned int len, unsigned int offset)
-{
-	return __bio_add_page(q, bio, page, len, offset,
-			      queue_max_hw_sectors(q));
-}
 EXPORT_SYMBOL(bio_add_pc_page);
 
 /**
@@ -834,16 +780,47 @@ EXPORT_SYMBOL(bio_add_pc_page);
  *	@len: vec entry length
  *	@offset: vec entry offset
  *
- *	Attempt to add a page to the bio_vec maplist. This can fail for a
- *	number of reasons, such as the bio being full or target block device
- *	limitations. The target block device must allow bio's up to PAGE_SIZE,
- *	so it is always possible to add a single page to an empty bio.
+ *	Attempt to add a page to the bio_vec maplist. This will only fail if
+ *	bio->bi_vcnt == bio->bi_max_vecs.
  */
-int bio_add_page(struct bio *bio, struct page *page, unsigned int len,
-		 unsigned int offset)
+int bio_add_page(struct bio *bio, struct page *page,
+		 unsigned int len, unsigned int offset)
 {
-	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
-	return __bio_add_page(q, bio, page, len, offset, queue_max_sectors(q));
+	struct bio_vec *bv;
+
+	/*
+	 * cloned bio must not modify vec list
+	 */
+	if (unlikely(bio_flagged(bio, BIO_CLONED)))
+		return 0;
+
+	/*
+	 * For filesystems with a blocksize smaller than the pagesize
+	 * we will often be called with the same page as last time and
+	 * a consecutive offset.  Optimize this special case.
+	 */
+	if (bio->bi_vcnt > 0) {
+		bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
+
+		if (page == bv->bv_page &&
+		    offset == bv->bv_offset + bv->bv_len) {
+			bv->bv_len += len;
+			goto done;
+		}
+	}
+
+	if (bio->bi_vcnt >= bio->bi_max_vecs)
+		return 0;
+
+	bv		= &bio->bi_io_vec[bio->bi_vcnt];
+	bv->bv_page	= page;
+	bv->bv_len	= len;
+	bv->bv_offset	= offset;
+
+	bio->bi_vcnt++;
+done:
+	bio->bi_iter.bi_size += len;
+	return len;
 }
 EXPORT_SYMBOL(bio_add_page);
 
-- 
1.8.4.4

[PATCH 03/11] btrfs: generic_make_request() handles arbitrary size bios now

[Kent Overstreet]

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
---
 fs/btrfs/volumes.c | 73 ------------------------------------------------------
 1 file changed, 73 deletions(-)

diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index 37972d5..d6dbe80 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -5391,34 +5391,6 @@ static noinline void btrfs_schedule_bio(struct btrfs_root *root,
 				   &device->work);
 }
 
-static int bio_size_ok(struct block_device *bdev, struct bio *bio,
-		       sector_t sector)
-{
-	struct bio_vec *prev;
-	struct request_queue *q = bdev_get_queue(bdev);
-	unsigned int max_sectors = queue_max_sectors(q);
-	struct bvec_merge_data bvm = {
-		.bi_bdev = bdev,
-		.bi_sector = sector,
-		.bi_rw = bio->bi_rw,
-	};
-
-	if (WARN_ON(bio->bi_vcnt == 0))
-		return 1;
-
-	prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
-	if (bio_sectors(bio) > max_sectors)
-		return 0;
-
-	if (!q->merge_bvec_fn)
-		return 1;
-
-	bvm.bi_size = bio->bi_iter.bi_size - prev->bv_len;
-	if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len)
-		return 0;
-	return 1;
-}
-
 static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,
 			      struct bio *bio, u64 physical, int dev_nr,
 			      int rw, int async)
@@ -5449,38 +5421,6 @@ static void submit_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,
 		btrfsic_submit_bio(rw, bio);
 }
 
-static int breakup_stripe_bio(struct btrfs_root *root, struct btrfs_bio *bbio,
-			      struct bio *first_bio, struct btrfs_device *dev,
-			      int dev_nr, int rw, int async)
-{
-	struct bio_vec *bvec = first_bio->bi_io_vec;
-	struct bio *bio;
-	int nr_vecs = bio_get_nr_vecs(dev->bdev);
-	u64 physical = bbio->stripes[dev_nr].physical;
-
-again:
-	bio = btrfs_bio_alloc(dev->bdev, physical >> 9, nr_vecs, GFP_NOFS);
-	if (!bio)
-		return -ENOMEM;
-
-	while (bvec <= (first_bio->bi_io_vec + first_bio->bi_vcnt - 1)) {
-		if (bio_add_page(bio, bvec->bv_page, bvec->bv_len,
-				 bvec->bv_offset) < bvec->bv_len) {
-			u64 len = bio->bi_iter.bi_size;
-
-			atomic_inc(&bbio->stripes_pending);
-			submit_stripe_bio(root, bbio, bio, physical, dev_nr,
-					  rw, async);
-			physical += len;
-			goto again;
-		}
-		bvec++;
-	}
-
-	submit_stripe_bio(root, bbio, bio, physical, dev_nr, rw, async);
-	return 0;
-}
-
 static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)
 {
 	atomic_inc(&bbio->error);
@@ -5549,19 +5489,6 @@ int btrfs_map_bio(struct btrfs_root *root, int rw, struct bio *bio,
 			continue;
 		}
 
-		/*
-		 * Check and see if we're ok with this bio based on it's size
-		 * and offset with the given device.
-		 */
-		if (!bio_size_ok(dev->bdev, first_bio,
-				 bbio->stripes[dev_nr].physical >> 9)) {
-			ret = breakup_stripe_bio(root, bbio, first_bio, dev,
-						 dev_nr, rw, async_submit);
-			BUG_ON(ret);
-			dev_nr++;
-			continue;
-		}
-
 		if (dev_nr < total_devs - 1) {
 			bio = btrfs_bio_clone(first_bio, GFP_NOFS);
 			BUG_ON(!bio); /* -ENOMEM */
-- 
1.8.4.4

[PATCH 04/11] btrfs: convert to bio_for_each_segment()

[Kent Overstreet]

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
---
 fs/btrfs/extent_io.c | 12 ++++++++---
 fs/btrfs/file-item.c | 59 ++++++++++++++++++++--------------------------------
 fs/btrfs/inode.c     | 22 +++++++-------------
 3 files changed, 39 insertions(+), 54 deletions(-)

diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index bcb6f1b..b6bf4fa 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -2579,12 +2579,18 @@ static int __must_check submit_one_bio(int rw, struct bio *bio,
 				       int mirror_num, unsigned long bio_flags)
 {
 	int ret = 0;
-	struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
-	struct page *page = bvec->bv_page;
+	struct bio_vec bvec = { 0 };
+	struct bvec_iter iter;
+	struct page *page;
 	struct extent_io_tree *tree = bio->bi_private;
 	u64 start;
 
-	start = page_offset(page) + bvec->bv_offset;
+	bio_for_each_segment(bvec, bio, iter)
+		if (bio_iter_last(bvec, iter))
+			break;
+
+	page = bvec.bv_page;
+	start = page_offset(page) + bvec.bv_offset;
 
 	bio->bi_private = NULL;
 
diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c
index 84a46a4..c0bcb78 100644
--- a/fs/btrfs/file-item.c
+++ b/fs/btrfs/file-item.c
@@ -162,7 +162,7 @@ static int __btrfs_lookup_bio_sums(struct btrfs_root *root,
 				   struct inode *inode, struct bio *bio,
 				   u64 logical_offset, u32 *dst, int dio)
 {
-	struct bio_vec *bvec = bio->bi_io_vec;
+	struct bvec_iter iter = bio->bi_iter;
 	struct btrfs_io_bio *btrfs_bio = btrfs_io_bio(bio);
 	struct btrfs_csum_item *item = NULL;
 	struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
@@ -171,10 +171,8 @@ static int __btrfs_lookup_bio_sums(struct btrfs_root *root,
 	u64 offset = 0;
 	u64 item_start_offset = 0;
 	u64 item_last_offset = 0;
-	u64 disk_bytenr;
 	u32 diff;
 	int nblocks;
-	int bio_index = 0;
 	int count;
 	u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy);
 
@@ -204,8 +202,6 @@ static int __btrfs_lookup_bio_sums(struct btrfs_root *root,
 	if (bio->bi_iter.bi_size > PAGE_CACHE_SIZE * 8)
 		path->reada = 2;
 
-	WARN_ON(bio->bi_vcnt <= 0);
-
 	/*
 	 * the free space stuff is only read when it hasn't been
 	 * updated in the current transaction.  So, we can safely
@@ -217,12 +213,13 @@ static int __btrfs_lookup_bio_sums(struct btrfs_root *root,
 		path->skip_locking = 1;
 	}
 
-	disk_bytenr = (u64)bio->bi_iter.bi_sector << 9;
 	if (dio)
 		offset = logical_offset;
-	while (bio_index < bio->bi_vcnt) {
+	while (iter.bi_size) {
+		u64 disk_bytenr = (u64)iter.bi_sector << 9;
+		struct bio_vec bvec = bio_iter_iovec(bio, iter);
 		if (!dio)
-			offset = page_offset(bvec->bv_page) + bvec->bv_offset;
+			offset = page_offset(bvec.bv_page) + bvec.bv_offset;
 		count = btrfs_find_ordered_sum(inode, offset, disk_bytenr,
 					       (u32 *)csum, nblocks);
 		if (count)
@@ -243,7 +240,7 @@ static int __btrfs_lookup_bio_sums(struct btrfs_root *root,
 				if (BTRFS_I(inode)->root->root_key.objectid ==
 				    BTRFS_DATA_RELOC_TREE_OBJECTID) {
 					set_extent_bits(io_tree, offset,
-						offset + bvec->bv_len - 1,
+						offset + bvec.bv_len - 1,
 						EXTENT_NODATASUM, GFP_NOFS);
 				} else {
 					printk(KERN_INFO "btrfs no csum found "
@@ -281,12 +278,9 @@ static int __btrfs_lookup_bio_sums(struct btrfs_root *root,
 found:
 		csum += count * csum_size;
 		nblocks -= count;
-		while (count--) {
-			disk_bytenr += bvec->bv_len;
-			offset += bvec->bv_len;
-			bio_index++;
-			bvec++;
-		}
+		bio_advance_iter(bio, &iter,
+				 count << inode->i_sb->s_blocksize_bits);
+		offset += count << inode->i_sb->s_blocksize_bits;
 	}
 	btrfs_free_path(path);
 	return 0;
@@ -439,14 +433,12 @@ int btrfs_csum_one_bio(struct btrfs_root *root, struct inode *inode,
 	struct btrfs_ordered_sum *sums;
 	struct btrfs_ordered_extent *ordered;
 	char *data;
-	struct bio_vec *bvec = bio->bi_io_vec;
-	int bio_index = 0;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
 	int index;
-	unsigned long total_bytes = 0;
 	unsigned long this_sum_bytes = 0;
 	u64 offset;
 
-	WARN_ON(bio->bi_vcnt <= 0);
 	sums = kzalloc(btrfs_ordered_sum_size(root, bio->bi_iter.bi_size),
 		       GFP_NOFS);
 	if (!sums)
@@ -458,53 +450,46 @@ int btrfs_csum_one_bio(struct btrfs_root *root, struct inode *inode,
 	if (contig)
 		offset = file_start;
 	else
-		offset = page_offset(bvec->bv_page) + bvec->bv_offset;
+		offset = page_offset(bio_page(bio)) + bio_offset(bio);
 
 	ordered = btrfs_lookup_ordered_extent(inode, offset);
 	BUG_ON(!ordered); /* Logic error */
 	sums->bytenr = (u64)bio->bi_iter.bi_sector << 9;
 	index = 0;
 
-	while (bio_index < bio->bi_vcnt) {
+	bio_for_each_segment(bvec, bio, iter) {
 		if (!contig)
-			offset = page_offset(bvec->bv_page) + bvec->bv_offset;
+			offset = page_offset(bvec.bv_page) + bvec.bv_offset;
 
 		if (offset >= ordered->file_offset + ordered->len ||
 		    offset < ordered->file_offset) {
-			unsigned long bytes_left;
 			sums->len = this_sum_bytes;
 			this_sum_bytes = 0;
 			btrfs_add_ordered_sum(inode, ordered, sums);
 			btrfs_put_ordered_extent(ordered);
 
-			bytes_left = bio->bi_iter.bi_size - total_bytes;
-
-			sums = kzalloc(btrfs_ordered_sum_size(root, bytes_left),
+			sums = kzalloc(btrfs_ordered_sum_size(root, iter.bi_size),
 				       GFP_NOFS);
 			BUG_ON(!sums); /* -ENOMEM */
-			sums->len = bytes_left;
+			sums->len = iter.bi_size;
 			ordered = btrfs_lookup_ordered_extent(inode, offset);
 			BUG_ON(!ordered); /* Logic error */
-			sums->bytenr = ((u64)bio->bi_iter.bi_sector << 9) +
-				       total_bytes;
+			sums->bytenr = ((u64)iter.bi_sector) << 9;
 			index = 0;
 		}
 
-		data = kmap_atomic(bvec->bv_page);
+		data = kmap_atomic(bvec.bv_page);
 		sums->sums[index] = ~(u32)0;
-		sums->sums[index] = btrfs_csum_data(data + bvec->bv_offset,
+		sums->sums[index] = btrfs_csum_data(data + bvec.bv_offset,
 						    sums->sums[index],
-						    bvec->bv_len);
+						    bvec.bv_len);
 		kunmap_atomic(data);
 		btrfs_csum_final(sums->sums[index],
 				 (char *)(sums->sums + index));
 
-		bio_index++;
 		index++;
-		total_bytes += bvec->bv_len;
-		this_sum_bytes += bvec->bv_len;
-		offset += bvec->bv_len;
-		bvec++;
+		offset += bvec.bv_len;
+		this_sum_bytes += bvec.bv_len;
 	}
 	this_sum_bytes = 0;
 	btrfs_add_ordered_sum(inode, ordered, sums);
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 7ab0e94..9463b92 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -6985,12 +6985,11 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
 	struct btrfs_root *root = BTRFS_I(inode)->root;
 	struct bio *bio;
 	struct bio *orig_bio = dip->orig_bio;
-	struct bio_vec *bvec = orig_bio->bi_io_vec;
+	struct bio_vec bvec;
+	struct bvec_iter iter;
 	u64 start_sector = orig_bio->bi_iter.bi_sector;
 	u64 file_offset = dip->logical_offset;
-	u64 submit_len = 0;
 	u64 map_length;
-	int nr_pages = 0;
 	int ret = 0;
 	int async_submit = 0;
 
@@ -7021,10 +7020,12 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
 	bio->bi_end_io = btrfs_end_dio_bio;
 	atomic_inc(&dip->pending_bios);
 
-	while (bvec <= (orig_bio->bi_io_vec + orig_bio->bi_vcnt - 1)) {
-		if (unlikely(map_length < submit_len + bvec->bv_len ||
-		    bio_add_page(bio, bvec->bv_page, bvec->bv_len,
-				 bvec->bv_offset) < bvec->bv_len)) {
+	bio_for_each_segment(bvec, orig_bio, iter) {
+		if (unlikely(map_length < bio->bi_iter.bi_size + bvec.bv_len ||
+		    bio_add_page(bio, bvec.bv_page, bvec.bv_len,
+				 bvec.bv_offset) < bvec.bv_len)) {
+			unsigned submit_len = bio->bi_iter.bi_size;
+
 			/*
 			 * inc the count before we submit the bio so
 			 * we know the end IO handler won't happen before
@@ -7044,9 +7045,6 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
 			start_sector += submit_len >> 9;
 			file_offset += submit_len;
 
-			submit_len = 0;
-			nr_pages = 0;
-
 			bio = btrfs_dio_bio_alloc(orig_bio->bi_bdev,
 						  start_sector, GFP_NOFS);
 			if (!bio)
@@ -7062,10 +7060,6 @@ static int btrfs_submit_direct_hook(int rw, struct btrfs_dio_private *dip,
 				bio_put(bio);
 				goto out_err;
 			}
-		} else {
-			submit_len += bvec->bv_len;
-			nr_pages++;
-			bvec++;
 		}
 	}
 
-- 
1.8.4.4

[PATCH 05/11] iov_iter: Move iov_iter to uio.h

[Kent Overstreet]

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
---
 include/linux/fs.h  | 32 --------------------------------
 include/linux/uio.h | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 32 deletions(-)

diff --git a/include/linux/fs.h b/include/linux/fs.h
index 121f11f..7a19f5d 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -290,38 +290,6 @@ struct page;
 struct address_space;
 struct writeback_control;
 
-struct iov_iter {
-	const struct iovec *iov;
-	unsigned long nr_segs;
-	size_t iov_offset;
-	size_t count;
-};
-
-size_t iov_iter_copy_from_user_atomic(struct page *page,
-		struct iov_iter *i, unsigned long offset, size_t bytes);
-size_t iov_iter_copy_from_user(struct page *page,
-		struct iov_iter *i, unsigned long offset, size_t bytes);
-void iov_iter_advance(struct iov_iter *i, size_t bytes);
-int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes);
-size_t iov_iter_single_seg_count(const struct iov_iter *i);
-
-static inline void iov_iter_init(struct iov_iter *i,
-			const struct iovec *iov, unsigned long nr_segs,
-			size_t count, size_t written)
-{
-	i->iov = iov;
-	i->nr_segs = nr_segs;
-	i->iov_offset = 0;
-	i->count = count + written;
-
-	iov_iter_advance(i, written);
-}
-
-static inline size_t iov_iter_count(struct iov_iter *i)
-{
-	return i->count;
-}
-
 /*
  * "descriptor" for what we're up to with a read.
  * This allows us to use the same read code yet
diff --git a/include/linux/uio.h b/include/linux/uio.h
index c55ce24..347d70c 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -9,14 +9,23 @@
 #ifndef __LINUX_UIO_H
 #define __LINUX_UIO_H
 
+#include <linux/kernel.h>
 #include <uapi/linux/uio.h>
 
+struct page;
 
 struct kvec {
 	void *iov_base; /* and that should *never* hold a userland pointer */
 	size_t iov_len;
 };
 
+struct iov_iter {
+	const struct iovec *iov;
+	unsigned long nr_segs;
+	size_t iov_offset;
+	size_t count;
+};
+
 /*
  * Total number of bytes covered by an iovec.
  *
@@ -34,8 +43,49 @@ static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs)
 	return ret;
 }
 
+static inline struct iovec iov_iter_iovec(const struct iov_iter *iter)
+{
+	return (struct iovec) {
+		.iov_base = iter->iov->iov_base + iter->iov_offset,
+		.iov_len = min(iter->count,
+			       iter->iov->iov_len - iter->iov_offset),
+	};
+}
+
+#define iov_for_each(iov, iter, start)				\
+	for (iter = (start);					\
+	     (iter).count &&					\
+	     ((iov = iov_iter_iovec(&(iter))), 1);		\
+	     iov_iter_advance(&(iter), (iov).iov_len))
+
 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to);
 
+size_t iov_iter_copy_from_user_atomic(struct page *page,
+		struct iov_iter *i, unsigned long offset, size_t bytes);
+size_t iov_iter_copy_from_user(struct page *page,
+		struct iov_iter *i, unsigned long offset, size_t bytes);
+void iov_iter_advance(struct iov_iter *i, size_t bytes);
+int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes);
+size_t iov_iter_single_seg_count(const struct iov_iter *i);
+
+static inline void iov_iter_init(struct iov_iter *i,
+			const struct iovec *iov, unsigned long nr_segs,
+			size_t count, size_t written)
+{
+	i->iov = iov;
+	i->nr_segs = nr_segs;
+	i->iov_offset = 0;
+	i->count = count + written;
+
+	iov_iter_advance(i, written);
+}
+
+static inline size_t iov_iter_count(struct iov_iter *i)
+{
+	return i->count;
+}
+
 int memcpy_fromiovec(unsigned char *kdata, struct iovec *iov, int len);
 int memcpy_toiovec(struct iovec *iov, unsigned char *kdata, int len);
+
 #endif
-- 
1.8.4.4

Re: [PATCH 05/11] iov_iter: Move iov_iter to uio.h

[Christoph Hellwig]

On Tue, Dec 03, 2013 at 02:00:11PM -0800, Kent Overstreet wrote:
> Signed-off-by: Kent Overstreet <kmo@daterainc.com>

Looks like a good enough cleanup on it's own, might be worth to
pick up for .14 independent of the dio changes.

Reviewed-by: Christoph Hellwig <hch@lst.de>

[PATCH 06/11] iov_iter: Kill iov_iter_single_seg_count()

[Kent Overstreet]

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
---
 fs/fuse/file.c      |  6 +++---
 include/linux/uio.h |  1 -
 mm/filemap.c        | 15 +--------------
 3 files changed, 4 insertions(+), 18 deletions(-)

diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index 7e70506..69175a1 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -996,7 +996,7 @@ static ssize_t fuse_fill_write_pages(struct fuse_req *req,
 		if (!tmp) {
 			unlock_page(page);
 			page_cache_release(page);
-			bytes = min(bytes, iov_iter_single_seg_count(ii));
+			bytes = min(bytes, iov_iter_iovec(ii).iov_len);
 			goto again;
 		}
 
@@ -1187,7 +1187,7 @@ static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
 static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
 					size_t max_size)
 {
-	return min(iov_iter_single_seg_count(ii), max_size);
+	return min(iov_iter_iovec(ii).iov_len, max_size);
 }
 
 static int fuse_get_user_pages(struct fuse_req *req, struct iov_iter *ii,
@@ -1261,7 +1261,7 @@ static inline int fuse_iter_npages(const struct iov_iter *ii_p)
 	while (iov_iter_count(&ii) && npages < FUSE_MAX_PAGES_PER_REQ) {
 		unsigned long user_addr = fuse_get_user_addr(&ii);
 		unsigned offset = user_addr & ~PAGE_MASK;
-		size_t frag_size = iov_iter_single_seg_count(&ii);
+		size_t frag_size = iov_iter_iovec(&ii).iov_len;
 
 		npages += (frag_size + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
 		iov_iter_advance(&ii, frag_size);
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 347d70c..2b99e0d 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -66,7 +66,6 @@ size_t iov_iter_copy_from_user(struct page *page,
 		struct iov_iter *i, unsigned long offset, size_t bytes);
 void iov_iter_advance(struct iov_iter *i, size_t bytes);
 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes);
-size_t iov_iter_single_seg_count(const struct iov_iter *i);
 
 static inline void iov_iter_init(struct iov_iter *i,
 			const struct iovec *iov, unsigned long nr_segs,
diff --git a/mm/filemap.c b/mm/filemap.c
index b7749a9..1a40f12 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2075,19 +2075,6 @@ int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
 EXPORT_SYMBOL(iov_iter_fault_in_readable);
 
 /*
- * Return the count of just the current iov_iter segment.
- */
-size_t iov_iter_single_seg_count(const struct iov_iter *i)
-{
-	const struct iovec *iov = i->iov;
-	if (i->nr_segs == 1)
-		return i->count;
-	else
-		return min(i->count, iov->iov_len - i->iov_offset);
-}
-EXPORT_SYMBOL(iov_iter_single_seg_count);
-
-/*
  * Performs necessary checks before doing a write
  *
  * Can adjust writing position or amount of bytes to write.
@@ -2375,7 +2362,7 @@ again:
 			 * once without a pagefault.
 			 */
 			bytes = min_t(unsigned long, PAGE_CACHE_SIZE - offset,
-						iov_iter_single_seg_count(i));
+				      iov_iter_iovec(i).iov_len);
 			goto again;
 		}
 		pos += copied;
-- 
1.8.4.4

Re: [PATCH 06/11] iov_iter: Kill iov_iter_single_seg_count()

[Christoph Hellwig]

Needs a rationale on why you think this is correct.  A blurb on
why you think it's useful would be helpful as well.

[PATCH 07/11] iov_iter: Kill written arg to iov_iter_init()

[Kent Overstreet]

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
---
 fs/btrfs/file.c     |  8 +++++---
 fs/cifs/file.c      |  4 ++--
 fs/fuse/file.c      | 11 ++++++-----
 include/linux/uio.h |  9 +++------
 mm/filemap.c        |  4 +++-
 5 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c
index 82d0342..273c429 100644
--- a/fs/btrfs/file.c
+++ b/fs/btrfs/file.c
@@ -1577,8 +1577,9 @@ static ssize_t __btrfs_direct_write(struct kiocb *iocb,
 		return written;
 
 	pos += written;
-	count -= written;
-	iov_iter_init(&i, iov, nr_segs, count, written);
+	iov_iter_init(&i, iov, nr_segs, count);
+	iov_iter_advance(&i, written);
+
 	written_buffered = __btrfs_buffered_write(file, &i, pos);
 	if (written_buffered < 0) {
 		err = written_buffered;
@@ -1693,7 +1694,8 @@ static ssize_t btrfs_file_aio_write(struct kiocb *iocb,
 	} else {
 		struct iov_iter i;
 
-		iov_iter_init(&i, iov, nr_segs, count, num_written);
+		iov_iter_init(&i, iov, nr_segs, count + num_written);
+		iov_iter_advance(&i, num_written);
 
 		num_written = __btrfs_buffered_write(file, &i, pos);
 		if (num_written > 0)
diff --git a/fs/cifs/file.c b/fs/cifs/file.c
index 5a5a872..dc11685 100644
--- a/fs/cifs/file.c
+++ b/fs/cifs/file.c
@@ -2416,7 +2416,7 @@ cifs_iovec_write(struct file *file, const struct iovec *iov,
 	else
 		pid = current->tgid;
 
-	iov_iter_init(&it, iov, nr_segs, len, 0);
+	iov_iter_init(&it, iov, nr_segs, len);
 	do {
 		size_t save_len;
 
@@ -2720,7 +2720,7 @@ cifs_readdata_to_iov(struct cifs_readdata *rdata, const struct iovec *iov,
 	unsigned int i;
 
 	/* set up iov_iter and advance to the correct offset */
-	iov_iter_init(&ii, iov, nr_segs, iov_length(iov, nr_segs), 0);
+	iov_iter_init(&ii, iov, nr_segs, iov_length(iov, nr_segs));
 	iov_iter_advance(&ii, pos);
 
 	*copied = 0;
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index 69175a1..ce45524 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -1135,9 +1135,10 @@ static ssize_t fuse_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
 			goto out;
 
 		pos += written;
-		count -= written;
 
-		iov_iter_init(&i, iov, nr_segs, count, written);
+		iov_iter_init(&i, iov, nr_segs, count);
+		iov_iter_advance(&i, written);
+
 		written_buffered = fuse_perform_write(file, mapping, &i, pos);
 		if (written_buffered < 0) {
 			err = written_buffered;
@@ -1157,7 +1158,7 @@ static ssize_t fuse_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
 		written += written_buffered;
 		iocb->ki_pos = pos + written_buffered;
 	} else {
-		iov_iter_init(&i, iov, nr_segs, count, 0);
+		iov_iter_init(&i, iov, nr_segs, count);
 		written = fuse_perform_write(file, mapping, &i, pos);
 		if (written >= 0)
 			iocb->ki_pos = pos + written;
@@ -1283,7 +1284,7 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, const struct iovec *iov,
 	struct fuse_req *req;
 	struct iov_iter ii;
 
-	iov_iter_init(&ii, iov, nr_segs, count, 0);
+	iov_iter_init(&ii, iov, nr_segs, count);
 
 	if (io->async)
 		req = fuse_get_req_for_background(fc, fuse_iter_npages(&ii));
@@ -2171,7 +2172,7 @@ static int fuse_ioctl_copy_user(struct page **pages, struct iovec *iov,
 	if (!bytes)
 		return 0;
 
-	iov_iter_init(&ii, iov, nr_segs, bytes, 0);
+	iov_iter_init(&ii, iov, nr_segs, bytes);
 
 	while (iov_iter_count(&ii)) {
 		struct page *page = pages[page_idx++];
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 2b99e0d..d653fee 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -67,16 +67,13 @@ size_t iov_iter_copy_from_user(struct page *page,
 void iov_iter_advance(struct iov_iter *i, size_t bytes);
 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes);
 
-static inline void iov_iter_init(struct iov_iter *i,
-			const struct iovec *iov, unsigned long nr_segs,
-			size_t count, size_t written)
+static inline void iov_iter_init(struct iov_iter *i, const struct iovec *iov,
+				 unsigned long nr_segs, size_t count)
 {
 	i->iov = iov;
 	i->nr_segs = nr_segs;
 	i->iov_offset = 0;
-	i->count = count + written;
-
-	iov_iter_advance(i, written);
+	i->count = count;
 }
 
 static inline size_t iov_iter_count(struct iov_iter *i)
diff --git a/mm/filemap.c b/mm/filemap.c
index 1a40f12..d48f39f 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -2387,7 +2387,9 @@ generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
 	ssize_t status;
 	struct iov_iter i;
 
-	iov_iter_init(&i, iov, nr_segs, count, written);
+	iov_iter_init(&i, iov, nr_segs, count + written);
+	iov_iter_advance(&i, written);
+
 	status = generic_perform_write(file, &i, pos);
 
 	if (likely(status >= 0)) {
-- 
1.8.4.4

Re: [PATCH 07/11] iov_iter: Kill written arg to iov_iter_init()

[Christoph Hellwig]

This looks like a sligh cleanup to me.  But please provide a good patch
description telling why you're doing it.

[PATCH 08/11] block: convert to iov_iter

[Kent Overstreet]

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
---
 block/blk-map.c        |  27 ++++++------
 block/scsi_ioctl.c     |  17 ++------
 drivers/scsi/sg.c      |  15 +++----
 fs/bio.c               | 113 +++++++++++++++++++++++++------------------------
 include/linux/bio.h    |   8 ++--
 include/linux/blkdev.h |   3 +-
 6 files changed, 86 insertions(+), 97 deletions(-)

diff --git a/block/blk-map.c b/block/blk-map.c
index ae4ae10..d4faf7b 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -5,7 +5,7 @@
 #include <linux/module.h>
 #include <linux/bio.h>
 #include <linux/blkdev.h>
-#include <scsi/sg.h>		/* for struct sg_iovec */
+#include <linux/uio.h>
 
 #include "blk.h"
 
@@ -188,20 +188,22 @@ EXPORT_SYMBOL(blk_rq_map_user);
  *    unmapping.
  */
 int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
-			struct rq_map_data *map_data, struct sg_iovec *iov,
-			int iov_count, unsigned int len, gfp_t gfp_mask)
+			struct rq_map_data *map_data, struct iov_iter *iter,
+			gfp_t gfp_mask)
 {
 	struct bio *bio;
-	int i, read = rq_data_dir(rq) == READ;
+	int read = rq_data_dir(rq) == READ;
 	int unaligned = 0;
+	struct iov_iter i;
+	struct iovec iov;
 
-	if (!iov || iov_count <= 0)
+	if (!iter || !iter->count)
 		return -EINVAL;
 
-	for (i = 0; i < iov_count; i++) {
-		unsigned long uaddr = (unsigned long)iov[i].iov_base;
+	iov_for_each(iov, i, *iter) {
+		unsigned long uaddr = (unsigned long) iov.iov_base;
 
-		if (!iov[i].iov_len)
+		if (!iov.iov_len)
 			return -EINVAL;
 
 		/*
@@ -211,16 +213,15 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
 			unaligned = 1;
 	}
 
-	if (unaligned || (q->dma_pad_mask & len) || map_data)
-		bio = bio_copy_user_iov(q, map_data, iov, iov_count, read,
-					gfp_mask);
+	if (unaligned || (q->dma_pad_mask & iter->count) || map_data)
+		bio = bio_copy_user_iov(q, map_data, iter, read, gfp_mask);
 	else
-		bio = bio_map_user_iov(q, NULL, iov, iov_count, read, gfp_mask);
+		bio = bio_map_user_iov(q, NULL, iter, read, gfp_mask);
 
 	if (IS_ERR(bio))
 		return PTR_ERR(bio);
 
-	if (bio->bi_iter.bi_size != len) {
+	if (bio->bi_iter.bi_size != iter->count) {
 		/*
 		 * Grab an extra reference to this bio, as bio_unmap_user()
 		 * expects to be able to drop it twice as it happens on the
diff --git a/block/scsi_ioctl.c b/block/scsi_ioctl.c
index 625e3e4..7ae554a 100644
--- a/block/scsi_ioctl.c
+++ b/block/scsi_ioctl.c
@@ -322,28 +322,19 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk,
 	}
 
 	if (hdr->iovec_count) {
-		size_t iov_data_len;
 		struct iovec *iov;
+		struct iov_iter i;
 
 		ret = rw_copy_check_uvector(-1, hdr->dxferp, hdr->iovec_count,
 					    0, NULL, &iov);
 		if (ret < 0)
 			goto out;
 
-		iov_data_len = ret;
-		ret = 0;
-
 		/* SG_IO howto says that the shorter of the two wins */
-		if (hdr->dxfer_len < iov_data_len) {
-			hdr->iovec_count = iov_shorten(iov,
-						       hdr->iovec_count,
-						       hdr->dxfer_len);
-			iov_data_len = hdr->dxfer_len;
-		}
+		iov_iter_init(&i, iov, hdr->iovec_count,
+			      min(ret, hdr->dxfer_len));
 
-		ret = blk_rq_map_user_iov(q, rq, NULL, (struct sg_iovec *) iov,
-					  hdr->iovec_count,
-					  iov_data_len, GFP_KERNEL);
+		ret = blk_rq_map_user_iov(q, rq, NULL, &i, GFP_KERNEL);
 		kfree(iov);
 	} else if (hdr->dxfer_len)
 		ret = blk_rq_map_user(q, rq, NULL, hdr->dxferp, hdr->dxfer_len,
diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c
index df5e961..5cc44b9 100644
--- a/drivers/scsi/sg.c
+++ b/drivers/scsi/sg.c
@@ -1695,22 +1695,19 @@ static int sg_start_req(Sg_request *srp, unsigned char *cmd)
 	}
 
 	if (iov_count) {
-		int len, size = sizeof(struct sg_iovec) * iov_count;
+		int size = sizeof(struct sg_iovec) * iov_count;
 		struct iovec *iov;
+		struct iov_iter i;
 
 		iov = memdup_user(hp->dxferp, size);
 		if (IS_ERR(iov))
 			return PTR_ERR(iov);
 
-		len = iov_length(iov, iov_count);
-		if (hp->dxfer_len < len) {
-			iov_count = iov_shorten(iov, iov_count, hp->dxfer_len);
-			len = hp->dxfer_len;
-		}
+		iov_iter_init(&i, iov, iov_count,
+			      min_t(size_t, hp->dxfer_len,
+				    iov_length(iov, iov_count)));
 
-		res = blk_rq_map_user_iov(q, rq, md, (struct sg_iovec *)iov,
-					  iov_count,
-					  len, GFP_ATOMIC);
+		res = blk_rq_map_user_iov(q, rq, md, &i, GFP_ATOMIC);
 		kfree(iov);
 	} else
 		res = blk_rq_map_user(q, rq, md, hp->dxferp,
diff --git a/fs/bio.c b/fs/bio.c
index da27557..a7e5789 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -969,18 +969,17 @@ void bio_copy_data(struct bio *dst, struct bio *src)
 EXPORT_SYMBOL(bio_copy_data);
 
 struct bio_map_data {
-	int nr_sgvecs;
 	int is_our_pages;
-	struct sg_iovec sgvecs[];
+	struct iov_iter iter;
+	struct iovec sgvecs[];
 };
 
 static void bio_set_map_data(struct bio_map_data *bmd, struct bio *bio,
-			     struct sg_iovec *iov, int iov_count,
-			     int is_our_pages)
+			     struct iov_iter *iter, int is_our_pages)
 {
-	memcpy(bmd->sgvecs, iov, sizeof(struct sg_iovec) * iov_count);
-	bmd->nr_sgvecs = iov_count;
 	bmd->is_our_pages = is_our_pages;
+	bmd->iter = *iter;
+	memcpy(bmd->sgvecs, iter->iov, sizeof(struct iovec) * iter->nr_segs);
 	bio->bi_private = bmd;
 }
 
@@ -995,33 +994,30 @@ static struct bio_map_data *bio_alloc_map_data(int nr_segs,
 		       sizeof(struct sg_iovec) * iov_count, gfp_mask);
 }
 
-static int __bio_copy_iov(struct bio *bio, struct sg_iovec *iov, int iov_count,
+static int __bio_copy_iov(struct bio *bio, struct iov_iter *iter,
 			  int to_user, int from_user, int do_free_page)
 {
 	int ret = 0, i;
 	struct bio_vec *bvec;
-	int iov_idx = 0;
-	unsigned int iov_off = 0;
+	struct iov_iter iov_iter = *iter;
 
 	bio_for_each_segment_all(bvec, bio, i) {
 		char *bv_addr = page_address(bvec->bv_page);
 		unsigned int bv_len = bvec->bv_len;
 
-		while (bv_len && iov_idx < iov_count) {
-			unsigned int bytes;
-			char __user *iov_addr;
-
-			bytes = min_t(unsigned int,
-				      iov[iov_idx].iov_len - iov_off, bv_len);
-			iov_addr = iov[iov_idx].iov_base + iov_off;
+		while (bv_len && iov_iter.count) {
+			struct iovec iov = iov_iter_iovec(&iov_iter);
+			unsigned int bytes = min_t(unsigned int, bv_len,
+						   iov.iov_len);
 
 			if (!ret) {
 				if (to_user)
-					ret = copy_to_user(iov_addr, bv_addr,
-							   bytes);
+					ret = copy_to_user(iov.iov_base,
+							   bv_addr, bytes);
 
 				if (from_user)
-					ret = copy_from_user(bv_addr, iov_addr,
+					ret = copy_from_user(bv_addr,
+							     iov.iov_base,
 							     bytes);
 
 				if (ret)
@@ -1030,13 +1026,7 @@ static int __bio_copy_iov(struct bio *bio, struct sg_iovec *iov, int iov_count,
 
 			bv_len -= bytes;
 			bv_addr += bytes;
-			iov_addr += bytes;
-			iov_off += bytes;
-
-			if (iov[iov_idx].iov_len == iov_off) {
-				iov_idx++;
-				iov_off = 0;
-			}
+			iov_iter_advance(&iov_iter, bytes);
 		}
 
 		if (do_free_page)
@@ -1065,7 +1055,7 @@ int bio_uncopy_user(struct bio *bio)
 		 * don't copy into a random user address space, just free.
 		 */
 		if (current->mm)
-			ret = __bio_copy_iov(bio, bmd->sgvecs, bmd->nr_sgvecs,
+			ret = __bio_copy_iov(bio, &bmd->iter,
 					     bio_data_dir(bio) == READ,
 					     0, bmd->is_our_pages);
 		else if (bmd->is_our_pages)
@@ -1093,7 +1083,7 @@ EXPORT_SYMBOL(bio_uncopy_user);
  */
 struct bio *bio_copy_user_iov(struct request_queue *q,
 			      struct rq_map_data *map_data,
-			      struct sg_iovec *iov, int iov_count,
+			      struct iov_iter *iter,
 			      int write_to_vm, gfp_t gfp_mask)
 {
 	struct bio_map_data *bmd;
@@ -1102,16 +1092,16 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
 	struct bio *bio;
 	int i, ret;
 	int nr_pages = 0;
-	unsigned int len = 0;
+	unsigned int len;
 	unsigned int offset = map_data ? map_data->offset & ~PAGE_MASK : 0;
 
-	for (i = 0; i < iov_count; i++) {
+	for (i = 0; i < iter->nr_segs; i++) {
 		unsigned long uaddr;
 		unsigned long end;
 		unsigned long start;
 
-		uaddr = (unsigned long)iov[i].iov_base;
-		end = (uaddr + iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+		uaddr = (unsigned long) iter->iov[i].iov_base;
+		end = (uaddr + iter->iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
 		start = uaddr >> PAGE_SHIFT;
 
 		/*
@@ -1121,13 +1111,12 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
 			return ERR_PTR(-EINVAL);
 
 		nr_pages += end - start;
-		len += iov[i].iov_len;
 	}
 
 	if (offset)
 		nr_pages++;
 
-	bmd = bio_alloc_map_data(nr_pages, iov_count, gfp_mask);
+	bmd = bio_alloc_map_data(nr_pages, iter->nr_segs, gfp_mask);
 	if (!bmd)
 		return ERR_PTR(-ENOMEM);
 
@@ -1144,7 +1133,12 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
 	if (map_data) {
 		nr_pages = 1 << map_data->page_order;
 		i = map_data->offset / PAGE_SIZE;
+	} else {
+		i = 0;
 	}
+
+	len = iter->count;
+
 	while (len) {
 		unsigned int bytes = PAGE_SIZE;
 
@@ -1186,12 +1180,12 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
 	 */
 	if ((!write_to_vm && (!map_data || !map_data->null_mapped)) ||
 	    (map_data && map_data->from_user)) {
-		ret = __bio_copy_iov(bio, iov, iov_count, 0, 1, 0);
+		ret = __bio_copy_iov(bio, iter, 0, 1, 0);
 		if (ret)
 			goto cleanup;
 	}
 
-	bio_set_map_data(bmd, bio, iov, iov_count, map_data ? 0 : 1);
+	bio_set_map_data(bmd, bio, iter, map_data ? 0 : 1);
 	return bio;
 cleanup:
 	if (!map_data)
@@ -1221,30 +1215,35 @@ struct bio *bio_copy_user(struct request_queue *q, struct rq_map_data *map_data,
 			  unsigned long uaddr, unsigned int len,
 			  int write_to_vm, gfp_t gfp_mask)
 {
-	struct sg_iovec iov;
+	struct iovec iov;
+	struct iov_iter i;
 
-	iov.iov_base = (void __user *)uaddr;
+	iov.iov_base = (void __user *) uaddr;
 	iov.iov_len = len;
 
-	return bio_copy_user_iov(q, map_data, &iov, 1, write_to_vm, gfp_mask);
+	iov_iter_init(&i, &iov, 1, len);
+
+	return bio_copy_user_iov(q, map_data, &i, write_to_vm, gfp_mask);
 }
 EXPORT_SYMBOL(bio_copy_user);
 
 static struct bio *__bio_map_user_iov(struct request_queue *q,
 				      struct block_device *bdev,
-				      struct sg_iovec *iov, int iov_count,
+				      struct iov_iter *iter,
 				      int write_to_vm, gfp_t gfp_mask)
 {
-	int i, j;
+	int j;
 	int nr_pages = 0;
 	struct page **pages;
 	struct bio *bio;
 	int cur_page = 0;
 	int ret, offset;
+	struct iov_iter i;
+	struct iovec iov;
 
-	for (i = 0; i < iov_count; i++) {
-		unsigned long uaddr = (unsigned long)iov[i].iov_base;
-		unsigned long len = iov[i].iov_len;
+	iov_for_each(iov, i, *iter) {
+		unsigned long uaddr = (unsigned long) iov.iov_base;
+		unsigned long len = iov.iov_len;
 		unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
 		unsigned long start = uaddr >> PAGE_SHIFT;
 
@@ -1274,9 +1273,9 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
 	if (!pages)
 		goto out;
 
-	for (i = 0; i < iov_count; i++) {
-		unsigned long uaddr = (unsigned long)iov[i].iov_base;
-		unsigned long len = iov[i].iov_len;
+	iov_for_each(iov, i, *iter) {
+		unsigned long uaddr = (unsigned long) iov.iov_base;
+		unsigned long len = iov.iov_len;
 		unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
 		unsigned long start = uaddr >> PAGE_SHIFT;
 		const int local_nr_pages = end - start;
@@ -1331,10 +1330,10 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
 	return bio;
 
  out_unmap:
-	for (i = 0; i < nr_pages; i++) {
-		if(!pages[i])
+	for (j = 0; j < nr_pages; j++) {
+		if(!pages[j])
 			break;
-		page_cache_release(pages[i]);
+		page_cache_release(pages[j]);
 	}
  out:
 	kfree(pages);
@@ -1358,12 +1357,15 @@ struct bio *bio_map_user(struct request_queue *q, struct block_device *bdev,
 			 unsigned long uaddr, unsigned int len, int write_to_vm,
 			 gfp_t gfp_mask)
 {
-	struct sg_iovec iov;
+	struct iovec iov;
+	struct iov_iter i;
 
-	iov.iov_base = (void __user *)uaddr;
+	iov.iov_base = (void __user *) uaddr;
 	iov.iov_len = len;
 
-	return bio_map_user_iov(q, bdev, &iov, 1, write_to_vm, gfp_mask);
+	iov_iter_init(&i, &iov, 1, len);
+
+	return bio_map_user_iov(q, bdev, &i, write_to_vm, gfp_mask);
 }
 EXPORT_SYMBOL(bio_map_user);
 
@@ -1380,13 +1382,12 @@ EXPORT_SYMBOL(bio_map_user);
  *	device. Returns an error pointer in case of error.
  */
 struct bio *bio_map_user_iov(struct request_queue *q, struct block_device *bdev,
-			     struct sg_iovec *iov, int iov_count,
+			     struct iov_iter *iter,
 			     int write_to_vm, gfp_t gfp_mask)
 {
 	struct bio *bio;
 
-	bio = __bio_map_user_iov(q, bdev, iov, iov_count, write_to_vm,
-				 gfp_mask);
+	bio = __bio_map_user_iov(q, bdev, iter, write_to_vm, gfp_mask);
 	if (IS_ERR(bio))
 		return bio;
 
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 7065452..068f199 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -372,11 +372,11 @@ extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
 extern int bio_get_nr_vecs(struct block_device *);
 extern struct bio *bio_map_user(struct request_queue *, struct block_device *,
 				unsigned long, unsigned int, int, gfp_t);
-struct sg_iovec;
+struct iov_iter;
 struct rq_map_data;
 extern struct bio *bio_map_user_iov(struct request_queue *,
 				    struct block_device *,
-				    struct sg_iovec *, int, int, gfp_t);
+				    struct iov_iter *, int, gfp_t);
 extern void bio_unmap_user(struct bio *);
 extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
 				gfp_t);
@@ -402,8 +402,8 @@ extern int bio_alloc_pages(struct bio *bio, gfp_t gfp);
 extern struct bio *bio_copy_user(struct request_queue *, struct rq_map_data *,
 				 unsigned long, unsigned int, int, gfp_t);
 extern struct bio *bio_copy_user_iov(struct request_queue *,
-				     struct rq_map_data *, struct sg_iovec *,
-				     int, int, gfp_t);
+				     struct rq_map_data *, struct iov_iter *,
+				     int, gfp_t);
 extern int bio_uncopy_user(struct bio *);
 void zero_fill_bio(struct bio *bio);
 extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 88d4e7b..33bec67 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -833,8 +833,7 @@ extern int blk_rq_map_user(struct request_queue *, struct request *,
 extern int blk_rq_unmap_user(struct bio *);
 extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
 extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
-			       struct rq_map_data *, struct sg_iovec *, int,
-			       unsigned int, gfp_t);
+			       struct rq_map_data *, struct iov_iter *, gfp_t);
 extern int blk_execute_rq(struct request_queue *, struct gendisk *,
 			  struct request *, int);
 extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
-- 
1.8.4.4

Re: [PATCH 08/11] block: convert to iov_iter

[Christoph Hellwig]

Both subject and patch description are severely lacking.

[PATCH 09/11] block: iov_count_pages()

[Kent Overstreet]

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
---
 fs/bio.c            | 42 ++++++------------------------------------
 include/linux/uio.h |  2 ++
 lib/iovec.c         | 30 ++++++++++++++++++++++++++++++
 3 files changed, 38 insertions(+), 36 deletions(-)

diff --git a/fs/bio.c b/fs/bio.c
index a7e5789..a660e3f 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -1095,23 +1095,9 @@ struct bio *bio_copy_user_iov(struct request_queue *q,
 	unsigned int len;
 	unsigned int offset = map_data ? map_data->offset & ~PAGE_MASK : 0;
 
-	for (i = 0; i < iter->nr_segs; i++) {
-		unsigned long uaddr;
-		unsigned long end;
-		unsigned long start;
-
-		uaddr = (unsigned long) iter->iov[i].iov_base;
-		end = (uaddr + iter->iov[i].iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
-		start = uaddr >> PAGE_SHIFT;
-
-		/*
-		 * Overflow, abort
-		 */
-		if (end < start)
-			return ERR_PTR(-EINVAL);
-
-		nr_pages += end - start;
-	}
+	nr_pages = iov_count_pages(iter, 0);
+	if (nr_pages < 0)
+		return ERR_PTR(nr_pages);
 
 	if (offset)
 		nr_pages++;
@@ -1241,25 +1227,9 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
 	struct iov_iter i;
 	struct iovec iov;
 
-	iov_for_each(iov, i, *iter) {
-		unsigned long uaddr = (unsigned long) iov.iov_base;
-		unsigned long len = iov.iov_len;
-		unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
-		unsigned long start = uaddr >> PAGE_SHIFT;
-
-		/*
-		 * Overflow, abort
-		 */
-		if (end < start)
-			return ERR_PTR(-EINVAL);
-
-		nr_pages += end - start;
-		/*
-		 * buffer must be aligned to at least hardsector size for now
-		 */
-		if (uaddr & queue_dma_alignment(q))
-			return ERR_PTR(-EINVAL);
-	}
+	nr_pages = iov_count_pages(iter, queue_dma_alignment(q));
+	if (nr_pages < 0)
+		return ERR_PTR(nr_pages);
 
 	if (!nr_pages)
 		return ERR_PTR(-EINVAL);
diff --git a/include/linux/uio.h b/include/linux/uio.h
index d653fee..d74c3451 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -60,6 +60,8 @@ static inline struct iovec iov_iter_iovec(const struct iov_iter *iter)
 
 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to);
 
+int iov_count_pages(const struct iov_iter *iter, unsigned align);
+
 size_t iov_iter_copy_from_user_atomic(struct page *page,
 		struct iov_iter *i, unsigned long offset, size_t bytes);
 size_t iov_iter_copy_from_user(struct page *page,
diff --git a/lib/iovec.c b/lib/iovec.c
index 454baa8..fa2c050 100644
--- a/lib/iovec.c
+++ b/lib/iovec.c
@@ -1,5 +1,7 @@
 #include <linux/uaccess.h>
 #include <linux/export.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
 #include <linux/uio.h>
 
 /*
@@ -51,3 +53,31 @@ int memcpy_toiovec(struct iovec *iov, unsigned char *kdata, int len)
 	return 0;
 }
 EXPORT_SYMBOL(memcpy_toiovec);
+
+int iov_count_pages(const struct iov_iter *iter, unsigned align)
+{
+	struct iov_iter i = *iter;
+	int nr_pages = 0;
+
+	while (iov_iter_count(&i)) {
+		unsigned long uaddr = (unsigned long) i.iov->iov_base +
+			i.iov_offset;
+		unsigned long len = i.iov->iov_len - i.iov_offset;
+
+		if ((uaddr & align) || (len & align))
+			return -EINVAL;
+
+		/*
+		 * Overflow, abort
+		 */
+		if (uaddr + len < uaddr)
+			return -EINVAL;
+
+		nr_pages += DIV_ROUND_UP(len + offset_in_page(uaddr),
+					 PAGE_SIZE);
+		iov_iter_advance(&i, len);
+	}
+
+	return nr_pages;
+}
+EXPORT_SYMBOL(iov_count_pages);
-- 
1.8.4.4

Re: [PATCH 09/11] block: iov_count_pages()

[Christoph Hellwig]

Needs a better subject, description and probably a comment describing
the function.

I don't think the name choice is a good one either as it checks for
alignment in addition to counting up the vectors.

[PATCH 10/11] block: Add bio_get_user_pages()

[Kent Overstreet]

This replaces some of the code that was in __bio_map_user_iov(), and
soon we're going to use this helper in the dio code.

Note that this relies on the recent change to make
generic_make_request() take arbitrary sized bios - we're not using
bio_add_page() here.

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
Cc: Jens Axboe <axboe@kernel.dk>
---
 fs/bio.c            | 130 +++++++++++++++++++++++++++-------------------------
 include/linux/bio.h |   2 +
 2 files changed, 70 insertions(+), 62 deletions(-)

diff --git a/fs/bio.c b/fs/bio.c
index a660e3f..c93ac07 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -1213,19 +1213,79 @@ struct bio *bio_copy_user(struct request_queue *q, struct rq_map_data *map_data,
 }
 EXPORT_SYMBOL(bio_copy_user);
 
+/**
+ * bio_get_user_pages - pin user pages and add them to a biovec
+ * @bio: bio to add pages to
+ * @uaddr: start of user address
+ * @len: length in bytes
+ * @write_to_vm: bool indicating writing to pages or not
+ *
+ * Pins pages for up to @len bytes and appends them to @bio's bvec array. May
+ * pin only part of the requested pages - @bio need not have room for all the
+ * pages and can already have had pages added to it.
+ *
+ * Returns the number of bytes from @len added to @bio.
+ */
+ssize_t bio_get_user_pages(struct bio *bio, struct iov_iter *i, int write_to_vm)
+{
+	while (bio->bi_vcnt < bio->bi_max_vecs && iov_iter_count(i)) {
+		struct iovec iov = iov_iter_iovec(i);
+		int ret;
+		unsigned nr_pages, bytes;
+		unsigned offset = offset_in_page(iov.iov_base);
+		struct bio_vec *bv;
+		struct page **pages;
+
+		nr_pages = min_t(size_t,
+				 DIV_ROUND_UP(iov.iov_len + offset, PAGE_SIZE),
+				 bio->bi_max_vecs - bio->bi_vcnt);
+
+		bv = &bio->bi_io_vec[bio->bi_vcnt];
+		pages = (void *) bv;
+
+		ret = get_user_pages_fast((unsigned long) iov.iov_base,
+					  nr_pages, write_to_vm, pages);
+		if (ret < 0) {
+			if (bio->bi_vcnt)
+				return 0;
+
+			return ret;
+		}
+
+		bio->bi_vcnt += ret;
+		bytes = ret * PAGE_SIZE - offset;
+
+		while (ret--) {
+			bv[ret].bv_page = pages[ret];
+			bv[ret].bv_len = PAGE_SIZE;
+			bv[ret].bv_offset = 0;
+		}
+
+		bv[0].bv_offset += offset;
+		bv[0].bv_len -= offset;
+
+		if (bytes > iov.iov_len) {
+			bio->bi_io_vec[bio->bi_vcnt - 1].bv_len -=
+				bytes - iov.iov_len;
+			bytes = iov.iov_len;
+		}
+
+		bio->bi_iter.bi_size += bytes;
+		iov_iter_advance(i, bytes);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(bio_get_user_pages);
+
 static struct bio *__bio_map_user_iov(struct request_queue *q,
 				      struct block_device *bdev,
 				      struct iov_iter *iter,
 				      int write_to_vm, gfp_t gfp_mask)
 {
-	int j;
+	ssize_t ret;
 	int nr_pages = 0;
-	struct page **pages;
 	struct bio *bio;
-	int cur_page = 0;
-	int ret, offset;
-	struct iov_iter i;
-	struct iovec iov;
 
 	nr_pages = iov_count_pages(iter, queue_dma_alignment(q));
 	if (nr_pages < 0)
@@ -1238,57 +1298,10 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
 	if (!bio)
 		return ERR_PTR(-ENOMEM);
 
-	ret = -ENOMEM;
-	pages = kcalloc(nr_pages, sizeof(struct page *), gfp_mask);
-	if (!pages)
+	ret = bio_get_user_pages(bio, iter, write_to_vm);
+	if (ret < 0)
 		goto out;
 
-	iov_for_each(iov, i, *iter) {
-		unsigned long uaddr = (unsigned long) iov.iov_base;
-		unsigned long len = iov.iov_len;
-		unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
-		unsigned long start = uaddr >> PAGE_SHIFT;
-		const int local_nr_pages = end - start;
-		const int page_limit = cur_page + local_nr_pages;
-
-		ret = get_user_pages_fast(uaddr, local_nr_pages,
-				write_to_vm, &pages[cur_page]);
-		if (ret < local_nr_pages) {
-			ret = -EFAULT;
-			goto out_unmap;
-		}
-
-		offset = uaddr & ~PAGE_MASK;
-		for (j = cur_page; j < page_limit; j++) {
-			unsigned int bytes = PAGE_SIZE - offset;
-
-			if (len <= 0)
-				break;
-			
-			if (bytes > len)
-				bytes = len;
-
-			/*
-			 * sorry...
-			 */
-			if (bio_add_pc_page(q, bio, pages[j], bytes, offset) <
-					    bytes)
-				break;
-
-			len -= bytes;
-			offset = 0;
-		}
-
-		cur_page = j;
-		/*
-		 * release the pages we didn't map into the bio, if any
-		 */
-		while (j < page_limit)
-			page_cache_release(pages[j++]);
-	}
-
-	kfree(pages);
-
 	/*
 	 * set data direction, and check if mapped pages need bouncing
 	 */
@@ -1299,14 +1312,7 @@ static struct bio *__bio_map_user_iov(struct request_queue *q,
 	bio->bi_flags |= (1 << BIO_USER_MAPPED);
 	return bio;
 
- out_unmap:
-	for (j = 0; j < nr_pages; j++) {
-		if(!pages[j])
-			break;
-		page_cache_release(pages[j]);
-	}
  out:
-	kfree(pages);
 	bio_put(bio);
 	return ERR_PTR(ret);
 }
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 068f199..52a07da 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -370,6 +370,8 @@ extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
 extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
 			   unsigned int, unsigned int);
 extern int bio_get_nr_vecs(struct block_device *);
+struct iov_iter;
+extern ssize_t bio_get_user_pages(struct bio *, struct iov_iter *, int);
 extern struct bio *bio_map_user(struct request_queue *, struct block_device *,
 				unsigned long, unsigned int, int, gfp_t);
 struct iov_iter;
-- 
1.8.4.4

Re: [PATCH 10/11] block: Add bio_get_user_pages()

[Christoph Hellwig]

On Tue, Dec 03, 2013 at 02:00:16PM -0800, Kent Overstreet wrote:
> This replaces some of the code that was in __bio_map_user_iov(), and
> soon we're going to use this helper in the dio code.
> 
> Note that this relies on the recent change to make
> generic_make_request() take arbitrary sized bios - we're not using
> bio_add_page() here.

__bio_map_user_iov was using bio_add_pc_page anyway.  I might
misremember some of the earlier patches, but how do you do arbitrary
splitting of BLOCK_PC bios?  Given that they come with a SCSI CDB
attached I can't see how we could split them at all.

Also when you factor code out and change behaviour please always
try to do that in two separate patches so that it's easier reviewable.

[PATCH 11/11] direct-io: Rewrite based on immutable biovecs

[Kent Overstreet]

Near total rewrite of fs/direct-io.c. This makes use of our new bio
splitting functionality, and the fact that generic_make_request() will
take arbitrary size bios - we allocate a bio, pin pages to it directly,
then call the getblocks() function to map it wherever the filesystem
tells us - splitting as needed.

Appears to pass xfstests with CONFIG_XFS_DEBUG=y (would appreciate testing from
someone who uses xfstests more than I).

Doesn't quite work with btrfs yet - when running xfstests it eventually get
stuck in an infinite loop somewhere. First thing in the kernel log is a warning
at fs/btrfs/ordered-data.c:288 in btrfs_add_ordered_sum() - if any of the btrfs
people want to take a look that would be a great help.

It _definitely_ needs review/auditing from someone intimately familiar with the
expected behaviour w.r.t. everything related to filesystem semantics and the
handling of the getblocks() call - dio_send_bio() is what primarily implements
that behaviour.

Signed-off-by: Kent Overstreet <kmo@daterainc.com>
---
 fs/direct-io.c | 1426 +++++++++++++++-----------------------------------------
 1 file changed, 384 insertions(+), 1042 deletions(-)

diff --git a/fs/direct-io.c b/fs/direct-io.c
index 160a548..00adc44 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -8,7 +8,7 @@
  * 04Jul2002	Andrew Morton
  *		Initial version
  * 11Sep2002	janetinc@us.ibm.com
- * 		added readv/writev support.
+ *		added readv/writev support.
  * 29Oct2002	Andrew Morton
  *		rewrote bio_add_page() support.
  * 30Oct2002	pbadari@us.ibm.com
@@ -39,187 +39,48 @@
 #include <linux/prefetch.h>
 #include <linux/aio.h>
 
-/*
- * How many user pages to map in one call to get_user_pages().  This determines
- * the size of a structure in the slab cache
- */
-#define DIO_PAGES	64
-
-/*
- * This code generally works in units of "dio_blocks".  A dio_block is
- * somewhere between the hard sector size and the filesystem block size.  it
- * is determined on a per-invocation basis.   When talking to the filesystem
- * we need to convert dio_blocks to fs_blocks by scaling the dio_block quantity
- * down by dio->blkfactor.  Similarly, fs-blocksize quantities are converted
- * to bio_block quantities by shifting left by blkfactor.
- *
- * If blkfactor is zero then the user's request was aligned to the filesystem's
- * blocksize.
- */
-
-/* dio_state only used in the submission path */
-
-struct dio_submit {
-	struct bio *bio;		/* bio under assembly */
-	unsigned blkbits;		/* doesn't change */
-	unsigned blkfactor;		/* When we're using an alignment which
-					   is finer than the filesystem's soft
-					   blocksize, this specifies how much
-					   finer.  blkfactor=2 means 1/4-block
-					   alignment.  Does not change */
-	unsigned start_zero_done;	/* flag: sub-blocksize zeroing has
-					   been performed at the start of a
-					   write */
-	int pages_in_io;		/* approximate total IO pages */
-	size_t	size;			/* total request size (doesn't change)*/
-	sector_t block_in_file;		/* Current offset into the underlying
-					   file in dio_block units. */
-	unsigned blocks_available;	/* At block_in_file.  changes */
-	int reap_counter;		/* rate limit reaping */
-	sector_t final_block_in_request;/* doesn't change */
-	unsigned first_block_in_page;	/* doesn't change, Used only once */
-	int boundary;			/* prev block is at a boundary */
-	get_block_t *get_block;		/* block mapping function */
-	dio_submit_t *submit_io;	/* IO submition function */
-
-	loff_t logical_offset_in_bio;	/* current first logical block in bio */
-	sector_t final_block_in_bio;	/* current final block in bio + 1 */
-	sector_t next_block_for_io;	/* next block to be put under IO,
-					   in dio_blocks units */
-
-	/*
-	 * Deferred addition of a page to the dio.  These variables are
-	 * private to dio_send_cur_page(), submit_page_section() and
-	 * dio_bio_add_page().
-	 */
-	struct page *cur_page;		/* The page */
-	unsigned cur_page_offset;	/* Offset into it, in bytes */
-	unsigned cur_page_len;		/* Nr of bytes at cur_page_offset */
-	sector_t cur_page_block;	/* Where it starts */
-	loff_t cur_page_fs_offset;	/* Offset in file */
-
-	/*
-	 * Page fetching state. These variables belong to dio_refill_pages().
-	 */
-	int curr_page;			/* changes */
-	int total_pages;		/* doesn't change */
-	unsigned long curr_user_address;/* changes */
-
-	/*
-	 * Page queue.  These variables belong to dio_refill_pages() and
-	 * dio_get_page().
-	 */
-	unsigned head;			/* next page to process */
-	unsigned tail;			/* last valid page + 1 */
-};
-
 /* dio_state communicated between submission path and end_io */
 struct dio {
-	int flags;			/* doesn't change */
-	int rw;
-	struct inode *inode;
-	loff_t i_size;			/* i_size when submitted */
-	dio_iodone_t *end_io;		/* IO completion function */
-
-	void *private;			/* copy from map_bh.b_private */
+	int		flags;		/* doesn't change */
+	int		rw;
+	struct inode	*inode;
+	loff_t		i_size;		/* i_size when submitted */
+	unsigned	i_blkbits;
 
 	/* BIO completion state */
-	spinlock_t bio_lock;		/* protects BIO fields below */
-	int page_errors;		/* errno from get_user_pages() */
-	int is_async;			/* is IO async ? */
+	int		page_error;	/* errno from get_user_pages() */
+	int		io_error;	/* IO error in completion path */
 	bool defer_completion;		/* defer AIO completion to workqueue? */
-	int io_error;			/* IO error in completion path */
-	unsigned long refcount;		/* direct_io_worker() and bios */
-	struct bio *bio_list;		/* singly linked via bi_private */
+	atomic_t	refcount;	/* direct_io_worker() and bios */
+
+	dio_iodone_t	*end_io;	/* IO completion function */
+	void		*private;	/* copy from map_bh.b_private */
 	struct task_struct *waiter;	/* waiting task (NULL if none) */
 
 	/* AIO related stuff */
-	struct kiocb *iocb;		/* kiocb */
-	ssize_t result;                 /* IO result */
-
-	/*
-	 * pages[] (and any fields placed after it) are not zeroed out at
-	 * allocation time.  Don't add new fields after pages[] unless you
-	 * wish that they not be zeroed.
-	 */
-	union {
-		struct page *pages[DIO_PAGES];	/* page buffer */
-		struct work_struct complete_work;/* deferred AIO completion */
-	};
-} ____cacheline_aligned_in_smp;
+	struct kiocb	*iocb;		/* kiocb */
+	ssize_t		result;		/* IO result */
+	struct work_struct complete_work;/* deferred AIO completion */
 
-static struct kmem_cache *dio_cache __read_mostly;
+	struct bio	bio;
+};
 
-/*
- * How many pages are in the queue?
- */
-static inline unsigned dio_pages_present(struct dio_submit *sdio)
-{
-	return sdio->tail - sdio->head;
-}
+#define DIO_WAKEUP	(1U << 31)
 
-/*
- * Go grab and pin some userspace pages.   Typically we'll get 64 at a time.
- */
-static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)
-{
-	int ret;
-	int nr_pages;
-
-	nr_pages = min(sdio->total_pages - sdio->curr_page, DIO_PAGES);
-	ret = get_user_pages_fast(
-		sdio->curr_user_address,		/* Where from? */
-		nr_pages,			/* How many pages? */
-		dio->rw == READ,		/* Write to memory? */
-		&dio->pages[0]);		/* Put results here */
-
-	if (ret < 0 && sdio->blocks_available && (dio->rw & WRITE)) {
-		struct page *page = ZERO_PAGE(0);
-		/*
-		 * A memory fault, but the filesystem has some outstanding
-		 * mapped blocks.  We need to use those blocks up to avoid
-		 * leaking stale data in the file.
-		 */
-		if (dio->page_errors == 0)
-			dio->page_errors = ret;
-		page_cache_get(page);
-		dio->pages[0] = page;
-		sdio->head = 0;
-		sdio->tail = 1;
-		ret = 0;
-		goto out;
-	}
+struct dio_mapping {
+	/* Low bit for READ|WRITE */
+	enum {
+		MAP_MAPPED	= 0,
+		MAP_NEW		= 2,
+		MAP_UNMAPPED	= 4,
+	} state;
 
-	if (ret >= 0) {
-		sdio->curr_user_address += ret * PAGE_SIZE;
-		sdio->curr_page += ret;
-		sdio->head = 0;
-		sdio->tail = ret;
-		ret = 0;
-	}
-out:
-	return ret;	
-}
-
-/*
- * Get another userspace page.  Returns an ERR_PTR on error.  Pages are
- * buffered inside the dio so that we can call get_user_pages() against a
- * decent number of pages, less frequently.  To provide nicer use of the
- * L1 cache.
- */
-static inline struct page *dio_get_page(struct dio *dio,
-		struct dio_submit *sdio)
-{
-	if (dio_pages_present(sdio) == 0) {
-		int ret;
+	struct block_device	*bdev;
+	loff_t			offset;
+	size_t			size;
+};
 
-		ret = dio_refill_pages(dio, sdio);
-		if (ret)
-			return ERR_PTR(ret);
-		BUG_ON(dio_pages_present(sdio) == 0);
-	}
-	return dio->pages[sdio->head++];
-}
+static struct bio_set *dio_pool __read_mostly;
 
 /**
  * dio_complete() - called when all DIO BIO I/O has been completed
@@ -233,37 +94,18 @@ static inline struct page *dio_get_page(struct dio *dio,
  * filesystems can use it to hold additional state between get_block calls and
  * dio_complete.
  */
-static ssize_t dio_complete(struct dio *dio, loff_t offset, ssize_t ret,
-		bool is_async)
+static ssize_t dio_complete(struct dio *dio, loff_t offset,
+			    ssize_t ret, bool is_async)
 {
-	ssize_t transferred = 0;
-
-	/*
-	 * AIO submission can race with bio completion to get here while
-	 * expecting to have the last io completed by bio completion.
-	 * In that case -EIOCBQUEUED is in fact not an error we want
-	 * to preserve through this call.
-	 */
-	if (ret == -EIOCBQUEUED)
-		ret = 0;
-
-	if (dio->result) {
-		transferred = dio->result;
-
-		/* Check for short read case */
-		if ((dio->rw == READ) && ((offset + transferred) > dio->i_size))
-			transferred = dio->i_size - offset;
-	}
-
 	if (ret == 0)
-		ret = dio->page_errors;
+		ret = dio->page_error;
 	if (ret == 0)
 		ret = dio->io_error;
 	if (ret == 0)
-		ret = transferred;
+		ret = dio->result;
 
 	if (dio->end_io && dio->result)
-		dio->end_io(dio->iocb, offset, transferred, dio->private);
+		dio->end_io(dio->iocb, offset, dio->result, dio->private);
 
 	inode_dio_done(dio->inode);
 	if (is_async) {
@@ -271,7 +113,7 @@ static ssize_t dio_complete(struct dio *dio, loff_t offset, ssize_t ret,
 			int err;
 
 			err = generic_write_sync(dio->iocb->ki_filp, offset,
-						 transferred);
+						 dio->result);
 			if (err < 0 && ret > 0)
 				ret = err;
 		}
@@ -279,7 +121,7 @@ static ssize_t dio_complete(struct dio *dio, loff_t offset, ssize_t ret,
 		aio_complete(dio->iocb, ret, 0);
 	}
 
-	kmem_cache_free(dio_cache, dio);
+	bio_put(&dio->bio);
 	return ret;
 }
 
@@ -290,57 +132,6 @@ static void dio_aio_complete_work(struct work_struct *work)
 	dio_complete(dio, dio->iocb->ki_pos, 0, true);
 }
 
-static int dio_bio_complete(struct dio *dio, struct bio *bio);
-
-/*
- * Asynchronous IO callback. 
- */
-static void dio_bio_end_aio(struct bio *bio, int error)
-{
-	struct dio *dio = bio->bi_private;
-	unsigned long remaining;
-	unsigned long flags;
-
-	/* cleanup the bio */
-	dio_bio_complete(dio, bio);
-
-	spin_lock_irqsave(&dio->bio_lock, flags);
-	remaining = --dio->refcount;
-	if (remaining == 1 && dio->waiter)
-		wake_up_process(dio->waiter);
-	spin_unlock_irqrestore(&dio->bio_lock, flags);
-
-	if (remaining == 0) {
-		if (dio->result && dio->defer_completion) {
-			INIT_WORK(&dio->complete_work, dio_aio_complete_work);
-			queue_work(dio->inode->i_sb->s_dio_done_wq,
-				   &dio->complete_work);
-		} else {
-			dio_complete(dio, dio->iocb->ki_pos, 0, true);
-		}
-	}
-}
-
-/*
- * The BIO completion handler simply queues the BIO up for the process-context
- * handler.
- *
- * During I/O bi_private points at the dio.  After I/O, bi_private is used to
- * implement a singly-linked list of completed BIOs, at dio->bio_list.
- */
-static void dio_bio_end_io(struct bio *bio, int error)
-{
-	struct dio *dio = bio->bi_private;
-	unsigned long flags;
-
-	spin_lock_irqsave(&dio->bio_lock, flags);
-	bio->bi_private = dio->bio_list;
-	dio->bio_list = bio;
-	if (--dio->refcount == 1 && dio->waiter)
-		wake_up_process(dio->waiter);
-	spin_unlock_irqrestore(&dio->bio_lock, flags);
-}
-
 /**
  * dio_end_io - handle the end io action for the given bio
  * @bio: The direct io bio thats being completed
@@ -353,187 +144,67 @@ static void dio_bio_end_io(struct bio *bio, int error)
 void dio_end_io(struct bio *bio, int error)
 {
 	struct dio *dio = bio->bi_private;
+	unsigned remaining;
 
-	if (dio->is_async)
-		dio_bio_end_aio(bio, error);
-	else
-		dio_bio_end_io(bio, error);
-}
-EXPORT_SYMBOL_GPL(dio_end_io);
-
-static inline void
-dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,
-	      struct block_device *bdev,
-	      sector_t first_sector, int nr_vecs)
-{
-	struct bio *bio;
-
-	/*
-	 * bio_alloc() is guaranteed to return a bio when called with
-	 * __GFP_WAIT and we request a valid number of vectors.
-	 */
-	bio = bio_alloc(GFP_KERNEL, nr_vecs);
-
-	bio->bi_bdev = bdev;
-	bio->bi_iter.bi_sector = first_sector;
-	if (dio->is_async)
-		bio->bi_end_io = dio_bio_end_aio;
-	else
-		bio->bi_end_io = dio_bio_end_io;
-
-	sdio->bio = bio;
-	sdio->logical_offset_in_bio = sdio->cur_page_fs_offset;
-}
-
-/*
- * In the AIO read case we speculatively dirty the pages before starting IO.
- * During IO completion, any of these pages which happen to have been written
- * back will be redirtied by bio_check_pages_dirty().
- *
- * bios hold a dio reference between submit_bio and ->end_io.
- */
-static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)
-{
-	struct bio *bio = sdio->bio;
-	unsigned long flags;
-
-	bio->bi_private = dio;
-
-	spin_lock_irqsave(&dio->bio_lock, flags);
-	dio->refcount++;
-	spin_unlock_irqrestore(&dio->bio_lock, flags);
+	if (error)
+		dio->io_error = -EIO;
 
-	if (dio->is_async && dio->rw == READ)
-		bio_set_pages_dirty(bio);
+	if (dio->rw == READ) {
+		bio_check_pages_dirty(bio);	/* transfers ownership */
+	} else {
+		struct bio_vec *bv;
+		int i;
 
-	if (sdio->submit_io)
-		sdio->submit_io(dio->rw, bio, dio->inode,
-			       sdio->logical_offset_in_bio);
-	else
-		submit_bio(dio->rw, bio);
+		bio_for_each_segment_all(bv, bio, i)
+			page_cache_release(bv->bv_page);
+		bio_put(bio);
+	}
 
-	sdio->bio = NULL;
-	sdio->boundary = 0;
-	sdio->logical_offset_in_bio = 0;
-}
+	remaining = atomic_dec_return(&dio->refcount);
 
-/*
- * Release any resources in case of a failure
- */
-static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)
-{
-	while (dio_pages_present(sdio))
-		page_cache_release(dio_get_page(dio, sdio));
+	if (remaining == DIO_WAKEUP)
+		wake_up_process(dio->waiter);
+	else if (!remaining) {
+		if (dio->defer_completion) {
+			INIT_WORK(&dio->complete_work, dio_aio_complete_work);
+			queue_work(dio->inode->i_sb->s_dio_done_wq,
+				   &dio->complete_work);
+		} else {
+			dio_complete(dio, dio->iocb->ki_pos, 0, true);
+		}
+	}
 }
+EXPORT_SYMBOL_GPL(dio_end_io);
 
-/*
- * Wait for the next BIO to complete.  Remove it and return it.  NULL is
- * returned once all BIOs have been completed.  This must only be called once
- * all bios have been issued so that dio->refcount can only decrease.  This
- * requires that that the caller hold a reference on the dio.
- */
-static struct bio *dio_await_one(struct dio *dio)
+static void dio_wait_completion(struct dio *dio)
 {
-	unsigned long flags;
-	struct bio *bio = NULL;
+	if (atomic_add_return(DIO_WAKEUP - 1, &dio->refcount) == DIO_WAKEUP)
+		return;
 
-	spin_lock_irqsave(&dio->bio_lock, flags);
+	while (1) {
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		if (atomic_read(&dio->refcount) == DIO_WAKEUP)
+			break;
 
-	/*
-	 * Wait as long as the list is empty and there are bios in flight.  bio
-	 * completion drops the count, maybe adds to the list, and wakes while
-	 * holding the bio_lock so we don't need set_current_state()'s barrier
-	 * and can call it after testing our condition.
-	 */
-	while (dio->refcount > 1 && dio->bio_list == NULL) {
-		__set_current_state(TASK_UNINTERRUPTIBLE);
-		dio->waiter = current;
-		spin_unlock_irqrestore(&dio->bio_lock, flags);
 		io_schedule();
-		/* wake up sets us TASK_RUNNING */
-		spin_lock_irqsave(&dio->bio_lock, flags);
-		dio->waiter = NULL;
-	}
-	if (dio->bio_list) {
-		bio = dio->bio_list;
-		dio->bio_list = bio->bi_private;
 	}
-	spin_unlock_irqrestore(&dio->bio_lock, flags);
-	return bio;
+	__set_current_state(TASK_RUNNING);
 }
 
 /*
- * Process one completed BIO.  No locks are held.
+ * Clean any dirty buffers in the blockdev mapping which alias newly-created
+ * file blocks.  Only called for S_ISREG files - blockdevs do not set buffer_new
  */
-static int dio_bio_complete(struct dio *dio, struct bio *bio)
+static void clean_blockdev_aliases(struct dio *dio, struct buffer_head *map_bh)
 {
-	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
-	struct bio_vec *bvec;
 	unsigned i;
+	unsigned nblocks;
 
-	if (!uptodate)
-		dio->io_error = -EIO;
-
-	if (dio->is_async && dio->rw == READ) {
-		bio_check_pages_dirty(bio);	/* transfers ownership */
-	} else {
-		bio_for_each_segment_all(bvec, bio, i) {
-			struct page *page = bvec->bv_page;
-
-			if (dio->rw == READ && !PageCompound(page))
-				set_page_dirty_lock(page);
-			page_cache_release(page);
-		}
-		bio_put(bio);
-	}
-	return uptodate ? 0 : -EIO;
-}
-
-/*
- * Wait on and process all in-flight BIOs.  This must only be called once
- * all bios have been issued so that the refcount can only decrease.
- * This just waits for all bios to make it through dio_bio_complete.  IO
- * errors are propagated through dio->io_error and should be propagated via
- * dio_complete().
- */
-static void dio_await_completion(struct dio *dio)
-{
-	struct bio *bio;
-	do {
-		bio = dio_await_one(dio);
-		if (bio)
-			dio_bio_complete(dio, bio);
-	} while (bio);
-}
+	nblocks = map_bh->b_size >> dio->i_blkbits;
 
-/*
- * A really large O_DIRECT read or write can generate a lot of BIOs.  So
- * to keep the memory consumption sane we periodically reap any completed BIOs
- * during the BIO generation phase.
- *
- * This also helps to limit the peak amount of pinned userspace memory.
- */
-static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)
-{
-	int ret = 0;
-
-	if (sdio->reap_counter++ >= 64) {
-		while (dio->bio_list) {
-			unsigned long flags;
-			struct bio *bio;
-			int ret2;
-
-			spin_lock_irqsave(&dio->bio_lock, flags);
-			bio = dio->bio_list;
-			dio->bio_list = bio->bi_private;
-			spin_unlock_irqrestore(&dio->bio_lock, flags);
-			ret2 = dio_bio_complete(dio, bio);
-			if (ret == 0)
-				ret = ret2;
-		}
-		sdio->reap_counter = 0;
-	}
-	return ret;
+	for (i = 0; i < nblocks; i++)
+		unmap_underlying_metadata(map_bh->b_bdev,
+					  map_bh->b_blocknr + i);
 }
 
 /*
@@ -572,513 +243,309 @@ static int dio_set_defer_completion(struct dio *dio)
 	return 0;
 }
 
-/*
- * Call into the fs to map some more disk blocks.  We record the current number
- * of available blocks at sdio->blocks_available.  These are in units of the
- * fs blocksize, (1 << inode->i_blkbits).
- *
- * The fs is allowed to map lots of blocks at once.  If it wants to do that,
- * it uses the passed inode-relative block number as the file offset, as usual.
- *
- * get_block() is passed the number of i_blkbits-sized blocks which direct_io
- * has remaining to do.  The fs should not map more than this number of blocks.
- *
- * If the fs has mapped a lot of blocks, it should populate bh->b_size to
- * indicate how much contiguous disk space has been made available at
- * bh->b_blocknr.
- *
- * If *any* of the mapped blocks are new, then the fs must set buffer_new().
- * This isn't very efficient...
- *
- * In the case of filesystem holes: the fs may return an arbitrarily-large
- * hole by returning an appropriate value in b_size and by clearing
- * buffer_mapped().  However the direct-io code will only process holes one
- * block at a time - it will repeatedly call get_block() as it walks the hole.
- */
-static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,
-			   struct buffer_head *map_bh)
+static int get_blocks(struct dio *dio, loff_t offset, size_t size,
+		      struct dio_mapping *map, get_block_t *get_block)
 {
-	int ret;
-	sector_t fs_startblk;	/* Into file, in filesystem-sized blocks */
-	sector_t fs_endblk;	/* Into file, in filesystem-sized blocks */
-	unsigned long fs_count;	/* Number of filesystem-sized blocks */
-	int create;
-	unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor;
+	struct buffer_head map_bh = { 0, };
+	int ret, create;
+	unsigned i_mask = (1 << dio->i_blkbits) - 1;
+	unsigned fs_offset = offset & i_mask;
+	sector_t fs_blocknr = offset >> dio->i_blkbits;
 
 	/*
-	 * If there was a memory error and we've overwritten all the
-	 * mapped blocks then we can now return that memory error
+	 * For writes inside i_size on a DIO_SKIP_HOLES filesystem we
+	 * forbid block creations: only overwrites are permitted.
+	 * We will return early to the caller once we see an
+	 * unmapped buffer head returned, and the caller will fall
+	 * back to buffered I/O.
+	 *
+	 * Otherwise the decision is left to the get_blocks method,
+	 * which may decide to handle it or also return an unmapped
+	 * buffer head.
 	 */
-	ret = dio->page_errors;
-	if (ret == 0) {
-		BUG_ON(sdio->block_in_file >= sdio->final_block_in_request);
-		fs_startblk = sdio->block_in_file >> sdio->blkfactor;
-		fs_endblk = (sdio->final_block_in_request - 1) >>
-					sdio->blkfactor;
-		fs_count = fs_endblk - fs_startblk + 1;
+	create = dio->rw & WRITE;
+	if (dio->flags & DIO_SKIP_HOLES) {
+		if (fs_blocknr < dio->i_size >> dio->i_blkbits)
+			create = 0;
+	}
 
-		map_bh->b_state = 0;
-		map_bh->b_size = fs_count << i_blkbits;
+	/* fs expects units of fs_blocks */
+	map_bh.b_size = size + fs_offset;
+	map_bh.b_size = round_up(map_bh.b_size, 1 << dio->i_blkbits);
 
-		/*
-		 * For writes inside i_size on a DIO_SKIP_HOLES filesystem we
-		 * forbid block creations: only overwrites are permitted.
-		 * We will return early to the caller once we see an
-		 * unmapped buffer head returned, and the caller will fall
-		 * back to buffered I/O.
-		 *
-		 * Otherwise the decision is left to the get_blocks method,
-		 * which may decide to handle it or also return an unmapped
-		 * buffer head.
-		 */
-		create = dio->rw & WRITE;
-		if (dio->flags & DIO_SKIP_HOLES) {
-			if (sdio->block_in_file < (i_size_read(dio->inode) >>
-							sdio->blkbits))
-				create = 0;
-		}
+	ret = get_block(dio->inode, fs_blocknr, &map_bh, create);
+	if (ret)
+		return ret;
 
-		ret = (*sdio->get_block)(dio->inode, fs_startblk,
-						map_bh, create);
+	/* Store for completion */
+	dio->private = map_bh.b_private;
 
-		/* Store for completion */
-		dio->private = map_bh->b_private;
+	if (ret == 0 && buffer_defer_completion(&map_bh))
+		ret = dio_set_defer_completion(dio);
 
-		if (ret == 0 && buffer_defer_completion(map_bh))
-			ret = dio_set_defer_completion(dio);
-	}
-	return ret;
-}
+	if (buffer_new(&map_bh))
+		clean_blockdev_aliases(dio, &map_bh);
 
-/*
- * There is no bio.  Make one now.
- */
-static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,
-		sector_t start_sector, struct buffer_head *map_bh)
-{
-	sector_t sector;
-	int ret, nr_pages;
+	if (!buffer_mapped(&map_bh))
+		map->state = MAP_UNMAPPED;
+	else if (buffer_new(&map_bh))
+		map->state = MAP_NEW;
+	else
+		map->state = MAP_MAPPED;
+
+#if 0
+	/* Previous DIO code only handled holes one block at a time */
+	if (map->state == MAP_UNMAPPED)
+		map_bh.b_size = 1 << dio->i_blkbits;
+
+#endif
+	BUG_ON(map_bh.b_size <= fs_offset);
+
+	map->bdev = map_bh.b_bdev;
+	map->offset = (map_bh.b_blocknr << dio->i_blkbits) +
+		fs_offset;
+	map->size = min(map_bh.b_size - fs_offset, size);
 
-	ret = dio_bio_reap(dio, sdio);
-	if (ret)
-		goto out;
-	sector = start_sector << (sdio->blkbits - 9);
-	nr_pages = min(sdio->pages_in_io, bio_get_nr_vecs(map_bh->b_bdev));
-	nr_pages = min(nr_pages, BIO_MAX_PAGES);
-	BUG_ON(nr_pages <= 0);
-	dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);
-	sdio->boundary = 0;
-out:
 	return ret;
 }
 
-/*
- * Attempt to put the current chunk of 'cur_page' into the current BIO.  If
- * that was successful then update final_block_in_bio and take a ref against
- * the just-added page.
- *
- * Return zero on success.  Non-zero means the caller needs to start a new BIO.
- */
-static inline int dio_bio_add_page(struct dio_submit *sdio)
+static void __dio_bio_submit(struct dio *dio, struct bio *bio,
+			     loff_t offset, dio_submit_t *submit_io)
 {
-	int ret;
+	/*
+	 * Read accounting is performed in submit_bio()
+	 */
+	if (dio->rw & WRITE)
+		task_io_account_write(bio->bi_iter.bi_size);
 
-	ret = bio_add_page(sdio->bio, sdio->cur_page,
-			sdio->cur_page_len, sdio->cur_page_offset);
-	if (ret == sdio->cur_page_len) {
-		/*
-		 * Decrement count only, if we are done with this page
-		 */
-		if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE)
-			sdio->pages_in_io--;
-		page_cache_get(sdio->cur_page);
-		sdio->final_block_in_bio = sdio->cur_page_block +
-			(sdio->cur_page_len >> sdio->blkbits);
-		ret = 0;
-	} else {
-		ret = 1;
-	}
-	return ret;
+	if (submit_io)
+		submit_io(dio->rw, bio, dio->inode,
+			  offset >> dio->i_blkbits);
+	else
+		submit_bio(dio->rw, bio);
 }
-		
+
 /*
- * Put cur_page under IO.  The section of cur_page which is described by
- * cur_page_offset,cur_page_len is put into a BIO.  The section of cur_page
- * starts on-disk at cur_page_block.
- *
- * We take a ref against the page here (on behalf of its presence in the bio).
+ * For reads we speculatively dirty the pages before starting IO. During IO
+ * completion, any of these pages which happen to have been written back will be
+ * redirtied by bio_check_pages_dirty().
  *
- * The caller of this function is responsible for removing cur_page from the
- * dio, and for dropping the refcount which came from that presence.
+ * bios hold a dio reference between submit_bio and ->end_io.
  */
-static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,
-		struct buffer_head *map_bh)
+static int dio_bio_submit(struct dio *dio, struct bio *bio,
+			  struct dio_mapping *map,
+			  loff_t offset, dio_submit_t *submit_io)
 {
-	int ret = 0;
+	struct bio *split;
 
-	if (sdio->bio) {
-		loff_t cur_offset = sdio->cur_page_fs_offset;
-		loff_t bio_next_offset = sdio->logical_offset_in_bio +
-			sdio->bio->bi_iter.bi_size;
+	split = bio_next_split(bio, map->size >> 9,
+			       GFP_KERNEL, fs_bio_set);
 
-		/*
-		 * See whether this new request is contiguous with the old.
-		 *
-		 * Btrfs cannot handle having logically non-contiguous requests
-		 * submitted.  For example if you have
-		 *
-		 * Logical:  [0-4095][HOLE][8192-12287]
-		 * Physical: [0-4095]      [4096-8191]
-		 *
-		 * We cannot submit those pages together as one BIO.  So if our
-		 * current logical offset in the file does not equal what would
-		 * be the next logical offset in the bio, submit the bio we
-		 * have.
-		 */
-		if (sdio->final_block_in_bio != sdio->cur_page_block ||
-		    cur_offset != bio_next_offset)
-			dio_bio_submit(dio, sdio);
-	}
+	if (split != bio)
+		bio_chain(split, bio);
+	else
+		atomic_inc(&bio->bi_remaining);
 
-	if (sdio->bio == NULL) {
-		ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
-		if (ret)
-			goto out;
-	}
+	split->bi_bdev = map->bdev;
+	split->bi_iter.bi_sector = map->offset >> 9;
 
-	if (dio_bio_add_page(sdio) != 0) {
-		dio_bio_submit(dio, sdio);
-		ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
-		if (ret == 0) {
-			ret = dio_bio_add_page(sdio);
-			BUG_ON(ret != 0);
-		}
-	}
-out:
-	return ret;
+	dio->result += map->size;
+
+	__dio_bio_submit(dio, bio, offset, submit_io);
+
+	return split == bio;
 }
 
-/*
- * An autonomous function to put a chunk of a page under deferred IO.
- *
- * The caller doesn't actually know (or care) whether this piece of page is in
- * a BIO, or is under IO or whatever.  We just take care of all possible 
- * situations here.  The separation between the logic of do_direct_IO() and
- * that of submit_page_section() is important for clarity.  Please don't break.
- *
- * The chunk of page starts on-disk at blocknr.
- *
- * We perform deferred IO, by recording the last-submitted page inside our
- * private part of the dio structure.  If possible, we just expand the IO
- * across that page here.
- *
- * If that doesn't work out then we put the old page into the bio and add this
- * page to the dio instead.
- */
-static inline int
-submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page,
-		    unsigned offset, unsigned len, sector_t blocknr,
-		    struct buffer_head *map_bh)
+static void dio_write_zeroes(struct dio *dio, struct bio *parent,
+			     struct block_device *bdev,
+			     sector_t sector, size_t size,
+			     loff_t offset, dio_submit_t *submit_io)
 {
-	int ret = 0;
+	unsigned pages = DIV_ROUND_UP(size, PAGE_SIZE);
+	struct bio *bio = bio_alloc(GFP_KERNEL, pages);
 
-	if (dio->rw & WRITE) {
-		/*
-		 * Read accounting is performed in submit_bio()
-		 */
-		task_io_account_write(len);
+	while (pages--) {
+		bio->bi_io_vec[pages].bv_page = ZERO_PAGE(0);
+		bio->bi_io_vec[pages].bv_len = PAGE_SIZE;
+		bio->bi_io_vec[pages].bv_offset = 0;
 	}
 
-	/*
-	 * Can we just grow the current page's presence in the dio?
-	 */
-	if (sdio->cur_page == page &&
-	    sdio->cur_page_offset + sdio->cur_page_len == offset &&
-	    sdio->cur_page_block +
-	    (sdio->cur_page_len >> sdio->blkbits) == blocknr) {
-		sdio->cur_page_len += len;
-		goto out;
-	}
-
-	/*
-	 * If there's a deferred page already there then send it.
-	 */
-	if (sdio->cur_page) {
-		ret = dio_send_cur_page(dio, sdio, map_bh);
-		page_cache_release(sdio->cur_page);
-		sdio->cur_page = NULL;
-		if (ret)
-			return ret;
-	}
+	bio->bi_bdev = bdev;
+	bio->bi_iter.bi_sector = sector;
+	bio->bi_iter.bi_size = size;
 
-	page_cache_get(page);		/* It is in dio */
-	sdio->cur_page = page;
-	sdio->cur_page_offset = offset;
-	sdio->cur_page_len = len;
-	sdio->cur_page_block = blocknr;
-	sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits;
-out:
-	/*
-	 * If sdio->boundary then we want to schedule the IO now to
-	 * avoid metadata seeks.
-	 */
-	if (sdio->boundary) {
-		ret = dio_send_cur_page(dio, sdio, map_bh);
-		dio_bio_submit(dio, sdio);
-		page_cache_release(sdio->cur_page);
-		sdio->cur_page = NULL;
-	}
-	return ret;
+	bio_chain(bio, parent);
+	__dio_bio_submit(dio, bio, offset, submit_io);
 }
 
-/*
- * Clean any dirty buffers in the blockdev mapping which alias newly-created
- * file blocks.  Only called for S_ISREG files - blockdevs do not set
- * buffer_new
- */
-static void clean_blockdev_aliases(struct dio *dio, struct buffer_head *map_bh)
+static void dio_zero_partial_block_front(struct dio *dio, struct bio *bio,
+					 struct dio_mapping *map, loff_t offset,
+					 dio_submit_t *submit_io)
 {
-	unsigned i;
-	unsigned nblocks;
-
-	nblocks = map_bh->b_size >> dio->inode->i_blkbits;
-
-	for (i = 0; i < nblocks; i++) {
-		unmap_underlying_metadata(map_bh->b_bdev,
-					  map_bh->b_blocknr + i);
-	}
+	unsigned blksize = 1 << dio->i_blkbits;
+	unsigned blkmask = blksize - 1;
+	unsigned front = offset & blkmask;
+
+	if (front)
+		dio_write_zeroes(dio, bio, map->bdev,
+				 (map->offset - front) >> 9,
+				 front, offset, submit_io);
 }
 
-/*
- * If we are not writing the entire block and get_block() allocated
- * the block for us, we need to fill-in the unused portion of the
- * block with zeros. This happens only if user-buffer, fileoffset or
- * io length is not filesystem block-size multiple.
- *
- * `end' is zero if we're doing the start of the IO, 1 at the end of the
- * IO.
- */
-static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio,
-		int end, struct buffer_head *map_bh)
+static void dio_zero_partial_block_back(struct dio *dio, struct bio *bio,
+					struct dio_mapping *map, loff_t offset,
+					dio_submit_t *submit_io)
 {
-	unsigned dio_blocks_per_fs_block;
-	unsigned this_chunk_blocks;	/* In dio_blocks */
-	unsigned this_chunk_bytes;
-	struct page *page;
+	unsigned blksize = 1 << dio->i_blkbits;
+	unsigned blkmask = blksize - 1;
+	unsigned back = (offset + map->size) & blkmask;
+
+	if (back)
+		dio_write_zeroes(dio, bio, map->bdev,
+				 (map->offset + map->size) >> 9,
+				 blksize - back, offset, submit_io);
+}
 
-	sdio->start_zero_done = 1;
-	if (!sdio->blkfactor || !buffer_new(map_bh))
-		return;
+static int dio_read_zeroes(struct dio *dio, struct bio *bio,
+			   struct dio_mapping *map)
+{
+	swap(bio->bi_iter.bi_size, map->size);
+	zero_fill_bio(bio);
+	swap(bio->bi_iter.bi_size, map->size);
 
-	dio_blocks_per_fs_block = 1 << sdio->blkfactor;
-	this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1);
+	dio->result += map->size;
+	bio_advance(bio, map->size);
 
-	if (!this_chunk_blocks)
-		return;
+	return !bio->bi_iter.bi_size;
+}
 
+static int dio_is_aligned(struct dio *dio, struct dio_mapping *map)
+{
 	/*
-	 * We need to zero out part of an fs block.  It is either at the
-	 * beginning or the end of the fs block.
+	 * XXX: have to make sure we're at least sector aligned, but maybe leave
+	 * the rest to generic_make_request()?
 	 */
-	if (end) 
-		this_chunk_blocks = dio_blocks_per_fs_block - this_chunk_blocks;
 
-	this_chunk_bytes = this_chunk_blocks << sdio->blkbits;
+	unsigned blocksize_mask =
+		roundup_pow_of_two(bdev_logical_block_size(map->bdev)) - 1;
 
-	page = ZERO_PAGE(0);
-	if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes,
-				sdio->next_block_for_io, map_bh))
-		return;
-
-	sdio->next_block_for_io += this_chunk_blocks;
+	return !(map->offset & blocksize_mask) &&
+		!(map->size & blocksize_mask);
 }
 
-/*
- * Walk the user pages, and the file, mapping blocks to disk and generating
- * a sequence of (page,offset,len,block) mappings.  These mappings are injected
- * into submit_page_section(), which takes care of the next stage of submission
- *
- * Direct IO against a blockdev is different from a file.  Because we can
- * happily perform page-sized but 512-byte aligned IOs.  It is important that
- * blockdev IO be able to have fine alignment and large sizes.
- *
- * So what we do is to permit the ->get_block function to populate bh.b_size
- * with the size of IO which is permitted at this offset and this i_blkbits.
- *
- * For best results, the blockdev should be set up with 512-byte i_blkbits and
- * it should set b_size to PAGE_SIZE or more inside get_block().  This gives
- * fine alignment but still allows this function to work in PAGE_SIZE units.
- */
-static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,
-			struct buffer_head *map_bh)
+static int dio_send_bio(struct dio *dio, struct bio *bio, loff_t offset,
+			get_block_t *get_block, dio_submit_t *submit_io)
 {
-	const unsigned blkbits = sdio->blkbits;
-	const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
-	struct page *page;
-	unsigned block_in_page;
-	int ret = 0;
-
-	/* The I/O can start at any block offset within the first page */
-	block_in_page = sdio->first_block_in_page;
-
-	while (sdio->block_in_file < sdio->final_block_in_request) {
-		page = dio_get_page(dio, sdio);
-		if (IS_ERR(page)) {
-			ret = PTR_ERR(page);
-			goto out;
-		}
+	struct dio_mapping map;
+	int ret = 0, rw = dio->rw & WRITE;
+	bool done;
 
-		while (block_in_page < blocks_per_page) {
-			unsigned offset_in_page = block_in_page << blkbits;
-			unsigned this_chunk_bytes;	/* # of bytes mapped */
-			unsigned this_chunk_blocks;	/* # of blocks */
-			unsigned u;
-
-			if (sdio->blocks_available == 0) {
-				/*
-				 * Need to go and map some more disk
-				 */
-				unsigned long blkmask;
-				unsigned long dio_remainder;
-
-				ret = get_more_blocks(dio, sdio, map_bh);
-				if (ret) {
-					page_cache_release(page);
-					goto out;
-				}
-				if (!buffer_mapped(map_bh))
-					goto do_holes;
-
-				sdio->blocks_available =
-						map_bh->b_size >> sdio->blkbits;
-				sdio->next_block_for_io =
-					map_bh->b_blocknr << sdio->blkfactor;
-				if (buffer_new(map_bh))
-					clean_blockdev_aliases(dio, map_bh);
-
-				if (!sdio->blkfactor)
-					goto do_holes;
-
-				blkmask = (1 << sdio->blkfactor) - 1;
-				dio_remainder = (sdio->block_in_file & blkmask);
-
-				/*
-				 * If we are at the start of IO and that IO
-				 * starts partway into a fs-block,
-				 * dio_remainder will be non-zero.  If the IO
-				 * is a read then we can simply advance the IO
-				 * cursor to the first block which is to be
-				 * read.  But if the IO is a write and the
-				 * block was newly allocated we cannot do that;
-				 * the start of the fs block must be zeroed out
-				 * on-disk
-				 */
-				if (!buffer_new(map_bh))
-					sdio->next_block_for_io += dio_remainder;
-				sdio->blocks_available -= dio_remainder;
-			}
-do_holes:
-			/* Handle holes */
-			if (!buffer_mapped(map_bh)) {
-				loff_t i_size_aligned;
-
-				/* AKPM: eargh, -ENOTBLK is a hack */
-				if (dio->rw & WRITE) {
-					page_cache_release(page);
-					return -ENOTBLK;
-				}
-
-				/*
-				 * Be sure to account for a partial block as the
-				 * last block in the file
-				 */
-				i_size_aligned = ALIGN(i_size_read(dio->inode),
-							1 << blkbits);
-				if (sdio->block_in_file >=
-						i_size_aligned >> blkbits) {
-					/* We hit eof */
-					page_cache_release(page);
-					goto out;
-				}
-				zero_user(page, block_in_page << blkbits,
-						1 << blkbits);
-				sdio->block_in_file++;
-				block_in_page++;
-				goto next_block;
+	if (rw == READ)
+		bio_set_pages_dirty(bio);
+
+	while (1) {
+		if (rw == READ && offset >= dio->i_size)
+			break;
+
+		ret = get_blocks(dio, offset, bio->bi_iter.bi_size,
+				 &map, get_block);
+		if (ret)
+			break;
+
+		switch (map.state|rw) {
+		case MAP_MAPPED|READ:
+		case MAP_MAPPED|WRITE:
+			if (!dio_is_aligned(dio, &map)) {
+				ret = -EINVAL;
+				goto out;
 			}
 
-			/*
-			 * If we're performing IO which has an alignment which
-			 * is finer than the underlying fs, go check to see if
-			 * we must zero out the start of this block.
-			 */
-			if (unlikely(sdio->blkfactor && !sdio->start_zero_done))
-				dio_zero_block(dio, sdio, 0, map_bh);
-
-			/*
-			 * Work out, in this_chunk_blocks, how much disk we
-			 * can add to this page
-			 */
-			this_chunk_blocks = sdio->blocks_available;
-			u = (PAGE_SIZE - offset_in_page) >> blkbits;
-			if (this_chunk_blocks > u)
-				this_chunk_blocks = u;
-			u = sdio->final_block_in_request - sdio->block_in_file;
-			if (this_chunk_blocks > u)
-				this_chunk_blocks = u;
-			this_chunk_bytes = this_chunk_blocks << blkbits;
-			BUG_ON(this_chunk_bytes == 0);
-
-			if (this_chunk_blocks == sdio->blocks_available)
-				sdio->boundary = buffer_boundary(map_bh);
-			ret = submit_page_section(dio, sdio, page,
-						  offset_in_page,
-						  this_chunk_bytes,
-						  sdio->next_block_for_io,
-						  map_bh);
-			if (ret) {
-				page_cache_release(page);
+			if (dio_bio_submit(dio, bio, &map, offset, submit_io))
+				goto out;
+			break;
+		case MAP_NEW|READ:
+		case MAP_UNMAPPED|READ:
+			if (dio_read_zeroes(dio, bio, &map))
+				goto out;
+
+			break;
+		case MAP_NEW|WRITE:
+			if (!dio_is_aligned(dio, &map)) {
+				ret = -EINVAL;
 				goto out;
 			}
-			sdio->next_block_for_io += this_chunk_blocks;
-
-			sdio->block_in_file += this_chunk_blocks;
-			block_in_page += this_chunk_blocks;
-			sdio->blocks_available -= this_chunk_blocks;
-next_block:
-			BUG_ON(sdio->block_in_file > sdio->final_block_in_request);
-			if (sdio->block_in_file == sdio->final_block_in_request)
-				break;
+
+			dio_zero_partial_block_front(dio, bio, &map,
+						     offset, submit_io);
+
+			done = dio_bio_submit(dio, bio, &map, offset, submit_io);
+
+			dio_zero_partial_block_back(dio, bio, &map,
+						    offset, submit_io);
+
+			if (done)
+				goto out;
+
+			break;
+		case MAP_UNMAPPED|WRITE:
+			/* AKPM: eargh, -ENOTBLK is a hack */
+			ret = -ENOTBLK;
+			goto out;
 		}
 
-		/* Drop the ref which was taken in get_user_pages() */
-		page_cache_release(page);
-		block_in_page = 0;
+		offset += map.size;
 	}
 out:
+	if (rw == READ && offset + dio->result > dio->i_size) {
+		BUG_ON(offset > dio->i_size ||
+		       (offset == dio->i_size && dio->result));
+		dio->result = dio->i_size - offset;
+	}
+
+	bio_endio(bio, 0);
 	return ret;
 }
 
-static inline int drop_refcount(struct dio *dio)
+static int dio_alloc_bios(struct dio *dio, loff_t offset,
+			  struct iov_iter *iter, unsigned nr_pages,
+			  get_block_t *get_block, dio_submit_t *submit_io)
 {
-	int ret2;
-	unsigned long flags;
+	ssize_t ret;
+	struct bio *bio;
 
-	/*
-	 * Sync will always be dropping the final ref and completing the
-	 * operation.  AIO can if it was a broken operation described above or
-	 * in fact if all the bios race to complete before we get here.  In
-	 * that case dio_complete() translates the EIOCBQUEUED into the proper
-	 * return code that the caller will hand to aio_complete().
-	 *
-	 * This is managed by the bio_lock instead of being an atomic_t so that
-	 * completion paths can drop their ref and use the remaining count to
-	 * decide to wake the submission path atomically.
-	 */
-	spin_lock_irqsave(&dio->bio_lock, flags);
-	ret2 = --dio->refcount;
-	spin_unlock_irqrestore(&dio->bio_lock, flags);
-	return ret2;
+	bio = &dio->bio;
+	bio_get(bio);
+	goto start;
+
+	while (iter->count) {
+		BUG_ON(!nr_pages);
+
+		bio = bio_alloc(GFP_KERNEL,
+				min_t(unsigned, BIO_MAX_PAGES, nr_pages));
+start:
+		bio->bi_private = dio;
+		bio->bi_end_io = dio_end_io;
+
+		ret = bio_get_user_pages(bio, iter, dio->rw == READ);
+		if (ret < 0) {
+			bio_put(bio);
+			dio->page_error = ret;
+			return 0;
+		}
+
+		nr_pages -= bio->bi_vcnt;
+
+		atomic_inc(&dio->refcount);
+		ret = dio_send_bio(dio, bio, offset + dio->result,
+				   get_block, submit_io);
+		if (ret)
+			return ret;
+
+		if (dio->rw == READ && offset + dio->result == dio->i_size)
+			break;
+	}
+
+	return 0;
 }
 
 /*
@@ -1100,82 +567,36 @@ static inline int drop_refcount(struct dio *dio)
  * expected that filesystem provide exclusion between new direct I/O
  * and truncates.  For DIO_LOCKING filesystems this is done by i_mutex,
  * but other filesystems need to take care of this on their own.
- *
- * NOTE: if you pass "sdio" to anything by pointer make sure that function
- * is always inlined. Otherwise gcc is unable to split the structure into
- * individual fields and will generate much worse code. This is important
- * for the whole file.
  */
-static inline ssize_t
+static ssize_t
 do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
-	struct block_device *bdev, const struct iovec *iov, loff_t offset, 
+	const struct iovec *iov, loff_t offset,
 	unsigned long nr_segs, get_block_t get_block, dio_iodone_t end_io,
 	dio_submit_t submit_io,	int flags)
 {
-	int seg;
-	size_t size;
-	unsigned long addr;
-	unsigned i_blkbits = ACCESS_ONCE(inode->i_blkbits);
-	unsigned blkbits = i_blkbits;
-	unsigned blocksize_mask = (1 << blkbits) - 1;
-	ssize_t retval = -EINVAL;
-	loff_t end = offset;
+	unsigned nr_pages = 0, i_blkbits;
+	size_t size = iocb->ki_nbytes;
+	ssize_t retval = 0;
 	struct dio *dio;
-	struct dio_submit sdio = { 0, };
-	unsigned long user_addr;
-	size_t bytes;
-	struct buffer_head map_bh = { 0, };
 	struct blk_plug plug;
+	struct iov_iter iter;
 
 	if (rw & WRITE)
 		rw = WRITE_ODIRECT;
 
-	/*
-	 * Avoid references to bdev if not absolutely needed to give
-	 * the early prefetch in the caller enough time.
-	 */
-
-	if (offset & blocksize_mask) {
-		if (bdev)
-			blkbits = blksize_bits(bdev_logical_block_size(bdev));
-		blocksize_mask = (1 << blkbits) - 1;
-		if (offset & blocksize_mask)
-			goto out;
-	}
-
-	/* Check the memory alignment.  Blocks cannot straddle pages */
-	for (seg = 0; seg < nr_segs; seg++) {
-		addr = (unsigned long)iov[seg].iov_base;
-		size = iov[seg].iov_len;
-		end += size;
-		if (unlikely((addr & blocksize_mask) ||
-			     (size & blocksize_mask))) {
-			if (bdev)
-				blkbits = blksize_bits(
-					 bdev_logical_block_size(bdev));
-			blocksize_mask = (1 << blkbits) - 1;
-			if ((addr & blocksize_mask) || (size & blocksize_mask))
-				goto out;
-		}
-	}
+	i_blkbits = ACCESS_ONCE(inode->i_blkbits);
 
 	/* watch out for a 0 len io from a tricksy fs */
-	if (rw == READ && end == offset)
+	if (rw == READ && !size)
 		return 0;
 
-	dio = kmem_cache_alloc(dio_cache, GFP_KERNEL);
-	retval = -ENOMEM;
-	if (!dio)
-		goto out;
-	/*
-	 * Believe it or not, zeroing out the page array caused a .5%
-	 * performance regression in a database benchmark.  So, we take
-	 * care to only zero out what's needed.
-	 */
-	memset(dio, 0, offsetof(struct dio, pages));
+	iov_iter_init(&iter, iov, nr_segs, size);
+
+	nr_pages = iov_count_pages(&iter, 511);
+	if (nr_pages < 0)
+		return nr_pages;
 
-	dio->flags = flags;
-	if (dio->flags & DIO_LOCKING) {
+	if (flags & DIO_LOCKING) {
 		if (rw == READ) {
 			struct address_space *mapping =
 					iocb->ki_filp->f_mapping;
@@ -1184,114 +605,42 @@ do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
 			mutex_lock(&inode->i_mutex);
 
 			retval = filemap_write_and_wait_range(mapping, offset,
-							      end - 1);
+							   offset + size - 1);
 			if (retval) {
 				mutex_unlock(&inode->i_mutex);
-				kmem_cache_free(dio_cache, dio);
-				goto out;
+				return retval;
 			}
 		}
 	}
 
 	/*
-	 * For file extending writes updating i_size before data
-	 * writeouts complete can expose uninitialized blocks. So
-	 * even for AIO, we need to wait for i/o to complete before
-	 * returning in this case.
-	 */
-	dio->is_async = !is_sync_kiocb(iocb) && !((rw & WRITE) &&
-		(end > i_size_read(inode)));
-	dio->inode = inode;
-	dio->rw = rw;
-
-	/*
-	 * For AIO O_(D)SYNC writes we need to defer completions to a workqueue
-	 * so that we can call ->fsync.
-	 */
-	if (dio->is_async && (rw & WRITE) &&
-	    ((iocb->ki_filp->f_flags & O_DSYNC) ||
-	     IS_SYNC(iocb->ki_filp->f_mapping->host))) {
-		retval = dio_set_defer_completion(dio);
-		if (retval) {
-			/*
-			 * We grab i_mutex only for reads so we don't have
-			 * to release it here
-			 */
-			kmem_cache_free(dio_cache, dio);
-			goto out;
-		}
-	}
-
-	/*
 	 * Will be decremented at I/O completion time.
 	 */
 	atomic_inc(&inode->i_dio_count);
 
-	retval = 0;
-	sdio.blkbits = blkbits;
-	sdio.blkfactor = i_blkbits - blkbits;
-	sdio.block_in_file = offset >> blkbits;
-
-	sdio.get_block = get_block;
-	dio->end_io = end_io;
-	sdio.submit_io = submit_io;
-	sdio.final_block_in_bio = -1;
-	sdio.next_block_for_io = -1;
-
-	dio->iocb = iocb;
-	dio->i_size = i_size_read(inode);
-
-	spin_lock_init(&dio->bio_lock);
-	dio->refcount = 1;
-
-	/*
-	 * In case of non-aligned buffers, we may need 2 more
-	 * pages since we need to zero out first and last block.
-	 */
-	if (unlikely(sdio.blkfactor))
-		sdio.pages_in_io = 2;
-
-	for (seg = 0; seg < nr_segs; seg++) {
-		user_addr = (unsigned long)iov[seg].iov_base;
-		sdio.pages_in_io +=
-			((user_addr + iov[seg].iov_len + PAGE_SIZE-1) /
-				PAGE_SIZE - user_addr / PAGE_SIZE);
-	}
+	dio = container_of(bio_alloc_bioset(GFP_KERNEL,
+				    min_t(unsigned, BIO_MAX_PAGES, nr_pages),
+				    dio_pool),
+			   struct dio, bio);
+
+	dio->flags	= flags;
+	dio->rw		= rw;
+	dio->inode	= inode;
+	dio->i_size	= i_size_read(inode);
+	dio->i_blkbits	= i_blkbits;
+	dio->end_io	= end_io;
+	dio->private	= NULL;
+	dio->page_error	= 0;
+	dio->io_error	= 0;
+	atomic_set(&dio->refcount, 1);
+	dio->waiter	= current;
+	dio->iocb	= iocb;
+	dio->result	= 0;
 
 	blk_start_plug(&plug);
 
-	for (seg = 0; seg < nr_segs; seg++) {
-		user_addr = (unsigned long)iov[seg].iov_base;
-		sdio.size += bytes = iov[seg].iov_len;
-
-		/* Index into the first page of the first block */
-		sdio.first_block_in_page = (user_addr & ~PAGE_MASK) >> blkbits;
-		sdio.final_block_in_request = sdio.block_in_file +
-						(bytes >> blkbits);
-		/* Page fetching state */
-		sdio.head = 0;
-		sdio.tail = 0;
-		sdio.curr_page = 0;
-
-		sdio.total_pages = 0;
-		if (user_addr & (PAGE_SIZE-1)) {
-			sdio.total_pages++;
-			bytes -= PAGE_SIZE - (user_addr & (PAGE_SIZE - 1));
-		}
-		sdio.total_pages += (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
-		sdio.curr_user_address = user_addr;
-
-		retval = do_direct_IO(dio, &sdio, &map_bh);
-
-		dio->result += iov[seg].iov_len -
-			((sdio.final_block_in_request - sdio.block_in_file) <<
-					blkbits);
-
-		if (retval) {
-			dio_cleanup(dio, &sdio);
-			break;
-		}
-	} /* end iovec loop */
+	retval = dio_alloc_bios(dio, offset, &iter, nr_pages,
+				get_block, submit_io);
 
 	if (retval == -ENOTBLK) {
 		/*
@@ -1300,33 +649,10 @@ do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
 		 */
 		retval = 0;
 	}
-	/*
-	 * There may be some unwritten disk at the end of a part-written
-	 * fs-block-sized block.  Go zero that now.
-	 */
-	dio_zero_block(dio, &sdio, 1, &map_bh);
-
-	if (sdio.cur_page) {
-		ssize_t ret2;
-
-		ret2 = dio_send_cur_page(dio, &sdio, &map_bh);
-		if (retval == 0)
-			retval = ret2;
-		page_cache_release(sdio.cur_page);
-		sdio.cur_page = NULL;
-	}
-	if (sdio.bio)
-		dio_bio_submit(dio, &sdio);
 
 	blk_finish_plug(&plug);
 
 	/*
-	 * It is possible that, we return short IO due to end of file.
-	 * In that case, we need to release all the pages we got hold on.
-	 */
-	dio_cleanup(dio, &sdio);
-
-	/*
 	 * All block lookups have been performed. For READ requests
 	 * we can let i_mutex go now that its achieved its purpose
 	 * of protecting us from looking up uninitialized blocks.
@@ -1342,19 +668,28 @@ do_blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
 	 * This had *better* be the only place that raises -EIOCBQUEUED.
 	 */
 	BUG_ON(retval == -EIOCBQUEUED);
-	if (dio->is_async && retval == 0 && dio->result &&
-	    ((rw == READ) || (dio->result == sdio.size)))
-		retval = -EIOCBQUEUED;
-
-	if (retval != -EIOCBQUEUED)
-		dio_await_completion(dio);
 
-	if (drop_refcount(dio) == 0) {
+	/*
+	 * For file extending writes updating i_size before data
+	 * writeouts complete can expose uninitialized blocks. So
+	 * even for AIO, we need to wait for i/o to complete before
+	 * returning in this case.
+	 */
+	if (!is_sync_kiocb(iocb) &&
+	    retval == 0 && dio->result &&
+	    ((rw == READ) ||
+	     (offset + size <= dio->i_size &&
+	      dio->result == size))) {
+		if (atomic_dec_and_test(&dio->refcount))
+			retval = dio_complete(dio, offset, retval, false);
+		else
+			retval = -EIOCBQUEUED;
+	} else {
+		dio_wait_completion(dio);
 		retval = dio_complete(dio, offset, retval, false);
-	} else
-		BUG_ON(retval != -EIOCBQUEUED);
+		BUG_ON(retval == -EIOCBQUEUED);
+	}
 
-out:
 	return retval;
 }
 
@@ -1376,16 +711,23 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
 	prefetch(bdev->bd_queue);
 	prefetch((char *)bdev->bd_queue + SMP_CACHE_BYTES);
 
-	return do_blockdev_direct_IO(rw, iocb, inode, bdev, iov, offset,
+	return do_blockdev_direct_IO(rw, iocb, inode, iov, offset,
 				     nr_segs, get_block, end_io,
 				     submit_io, flags);
 }
-
 EXPORT_SYMBOL(__blockdev_direct_IO);
 
 static __init int dio_init(void)
 {
-	dio_cache = KMEM_CACHE(dio, SLAB_PANIC);
+	/*
+	 * First argument to bioset_create() is completely arbitrary - it just
+	 * has to be nonzero to always make forward progress.
+	 */
+
+	dio_pool = bioset_create(4, offsetof(struct dio, bio));
+	if (!dio_pool)
+		panic("dio: can't allocate bios\n");
+
 	return 0;
 }
-module_init(dio_init)
+subsys_initcall(dio_init);
-- 
1.8.4.4