* [RFC PATCH 13/28] block: Generalize bvec_should_split()
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
bvec_should_split() will need to also operate on dma_vecs so
generalize it to take an offset and length instead of a bio_vec.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
block/blk-merge.c | 31 +++++++++++++++++--------------
1 file changed, 17 insertions(+), 14 deletions(-)
diff --git a/block/blk-merge.c b/block/blk-merge.c
index d9e89c0ad40d..32653fca53ce 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -206,23 +206,25 @@ struct blk_segment_split_ctx {
unsigned sectors;
bool prv_valid;
- struct bio_vec bvprv;
+ unsigned prv_offset;
+ unsigned prv_len;
const unsigned max_sectors;
const unsigned max_segs;
};
-static bool bvec_should_split(struct request_queue *q, struct bio_vec *bv,
- struct blk_segment_split_ctx *ctx)
+static bool vec_should_split(struct request_queue *q, unsigned offset,
+ unsigned len, struct blk_segment_split_ctx *ctx)
{
/*
* If the queue doesn't support SG gaps and adding this
* offset would create a gap, disallow it.
*/
- if (ctx->prv_valid && bvec_gap_to_prev(q, &ctx->bvprv, bv->bv_offset))
+ if (ctx->prv_valid &&
+ vec_gap_to_prev(q, ctx->prv_offset, ctx->prv_len, offset))
return true;
- if (ctx->sectors + (bv->bv_len >> 9) > ctx->max_sectors) {
+ if (ctx->sectors + (len >> 9) > ctx->max_sectors) {
/*
* Consider this a new segment if we're splitting in
* the middle of this vector.
@@ -230,9 +232,9 @@ static bool bvec_should_split(struct request_queue *q, struct bio_vec *bv,
if (ctx->nsegs < ctx->max_segs &&
ctx->sectors < ctx->max_sectors) {
/* split in the middle of bvec */
- bv->bv_len = (ctx->max_sectors - ctx->sectors) << 9;
- bvec_split_segs(q, bv, &ctx->nsegs,
- &ctx->sectors, ctx->max_segs);
+ len = (ctx->max_sectors - ctx->sectors) << 9;
+ vec_split_segs(q, offset, len, &ctx->nsegs,
+ &ctx->sectors, ctx->max_segs);
}
return true;
}
@@ -240,14 +242,15 @@ static bool bvec_should_split(struct request_queue *q, struct bio_vec *bv,
if (ctx->nsegs == ctx->max_segs)
return true;
- ctx->bvprv = *bv;
+ ctx->prv_offset = offset;
+ ctx->prv_len = len;
ctx->prv_valid = true;
- if (bv->bv_offset + bv->bv_len <= PAGE_SIZE) {
+ if (offset + len <= PAGE_SIZE) {
ctx->nsegs++;
- ctx->sectors += bv->bv_len >> 9;
- } else if (bvec_split_segs(q, bv, &ctx->nsegs, &ctx->sectors,
- ctx->max_segs)) {
+ ctx->sectors += len >> 9;
+ } else if (vec_split_segs(q, offset, len, &ctx->nsegs, &ctx->sectors,
+ ctx->max_segs)) {
return true;
}
@@ -270,7 +273,7 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
};
bio_for_each_bvec(bv, bio, iter) {
- do_split = bvec_should_split(q, &bv, &ctx);
+ do_split = vec_should_split(q, bv.bv_offset, bv.bv_len, &ctx);
if (do_split)
break;
}
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 12/28] block: Create helper for bvec_should_split()
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
In order to support dma-direct bios, blk_bio_segment_split() will
need to operate on both bio_vecs and dma_vecs. In order to do
this, the code inside bio_for_each_bvec() is moved into a generic
helper called bvec_should_split().
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
block/blk-merge.c | 86 +++++++++++++++++++++++++----------------------
1 file changed, 46 insertions(+), 40 deletions(-)
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 414e61a714bf..d9e89c0ad40d 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -212,6 +212,48 @@ struct blk_segment_split_ctx {
const unsigned max_segs;
};
+static bool bvec_should_split(struct request_queue *q, struct bio_vec *bv,
+ struct blk_segment_split_ctx *ctx)
+{
+ /*
+ * If the queue doesn't support SG gaps and adding this
+ * offset would create a gap, disallow it.
+ */
+ if (ctx->prv_valid && bvec_gap_to_prev(q, &ctx->bvprv, bv->bv_offset))
+ return true;
+
+ if (ctx->sectors + (bv->bv_len >> 9) > ctx->max_sectors) {
+ /*
+ * Consider this a new segment if we're splitting in
+ * the middle of this vector.
+ */
+ if (ctx->nsegs < ctx->max_segs &&
+ ctx->sectors < ctx->max_sectors) {
+ /* split in the middle of bvec */
+ bv->bv_len = (ctx->max_sectors - ctx->sectors) << 9;
+ bvec_split_segs(q, bv, &ctx->nsegs,
+ &ctx->sectors, ctx->max_segs);
+ }
+ return true;
+ }
+
+ if (ctx->nsegs == ctx->max_segs)
+ return true;
+
+ ctx->bvprv = *bv;
+ ctx->prv_valid = true;
+
+ if (bv->bv_offset + bv->bv_len <= PAGE_SIZE) {
+ ctx->nsegs++;
+ ctx->sectors += bv->bv_len >> 9;
+ } else if (bvec_split_segs(q, bv, &ctx->nsegs, &ctx->sectors,
+ ctx->max_segs)) {
+ return true;
+ }
+
+ return false;
+}
+
static struct bio *blk_bio_segment_split(struct request_queue *q,
struct bio *bio,
struct bio_set *bs,
@@ -219,7 +261,7 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
{
struct bio_vec bv;
struct bvec_iter iter;
- bool do_split = true;
+ bool do_split = false;
struct bio *new = NULL;
struct blk_segment_split_ctx ctx = {
@@ -228,47 +270,11 @@ static struct bio *blk_bio_segment_split(struct request_queue *q,
};
bio_for_each_bvec(bv, bio, iter) {
- /*
- * If the queue doesn't support SG gaps and adding this
- * offset would create a gap, disallow it.
- */
- if (ctx.prv_valid && bvec_gap_to_prev(q, &ctx.bvprv,
- bv.bv_offset))
- goto split;
-
- if (ctx.sectors + (bv.bv_len >> 9) > ctx.max_sectors) {
- /*
- * Consider this a new segment if we're splitting in
- * the middle of this vector.
- */
- if (ctx.nsegs < ctx.max_segs &&
- ctx.sectors < ctx.max_sectors) {
- /* split in the middle of bvec */
- bv.bv_len =
- (ctx.max_sectors - ctx.sectors) << 9;
- bvec_split_segs(q, &bv, &ctx.nsegs,
- &ctx.sectors, ctx.max_segs);
- }
- goto split;
- }
-
- if (ctx.nsegs == ctx.max_segs)
- goto split;
-
- ctx.bvprv = bv;
- ctx.prv_valid = true;
-
- if (bv.bv_offset + bv.bv_len <= PAGE_SIZE) {
- ctx.nsegs++;
- ctx.sectors += bv.bv_len >> 9;
- } else if (bvec_split_segs(q, &bv, &ctx.nsegs, &ctx.sectors,
- ctx.max_segs)) {
- goto split;
- }
+ do_split = bvec_should_split(q, &bv, &ctx);
+ if (do_split)
+ break;
}
- do_split = false;
-split:
*segs = ctx.nsegs;
if (do_split) {
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 11/28] block: Create blk_segment_split_ctx
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
In order to support dma-direct bios, blk_bio_segment_split() will
need to operate on both bio_vecs and dma_vecs. In order to do
this the code inside bio_for_each_bvec() needs to be moved into
a generic helper. Step one to do this is to put some of the
variables used inside the loop into a context structure so we
don't need to pass a dozen variables to this new function.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
block/blk-merge.c | 55 ++++++++++++++++++++++++++++++-----------------
1 file changed, 35 insertions(+), 20 deletions(-)
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 3581c7ac3c1b..414e61a714bf 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -201,63 +201,78 @@ static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv,
sectors, max_segs);
}
+struct blk_segment_split_ctx {
+ unsigned nsegs;
+ unsigned sectors;
+
+ bool prv_valid;
+ struct bio_vec bvprv;
+
+ const unsigned max_sectors;
+ const unsigned max_segs;
+};
+
static struct bio *blk_bio_segment_split(struct request_queue *q,
struct bio *bio,
struct bio_set *bs,
unsigned *segs)
{
- struct bio_vec bv, bvprv, *bvprvp = NULL;
+ struct bio_vec bv;
struct bvec_iter iter;
- unsigned nsegs = 0, sectors = 0;
bool do_split = true;
struct bio *new = NULL;
- const unsigned max_sectors = get_max_io_size(q, bio);
- const unsigned max_segs = queue_max_segments(q);
+
+ struct blk_segment_split_ctx ctx = {
+ .max_sectors = get_max_io_size(q, bio),
+ .max_segs = queue_max_segments(q),
+ };
bio_for_each_bvec(bv, bio, iter) {
/*
* If the queue doesn't support SG gaps and adding this
* offset would create a gap, disallow it.
*/
- if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset))
+ if (ctx.prv_valid && bvec_gap_to_prev(q, &ctx.bvprv,
+ bv.bv_offset))
goto split;
- if (sectors + (bv.bv_len >> 9) > max_sectors) {
+ if (ctx.sectors + (bv.bv_len >> 9) > ctx.max_sectors) {
/*
* Consider this a new segment if we're splitting in
* the middle of this vector.
*/
- if (nsegs < max_segs &&
- sectors < max_sectors) {
+ if (ctx.nsegs < ctx.max_segs &&
+ ctx.sectors < ctx.max_sectors) {
/* split in the middle of bvec */
- bv.bv_len = (max_sectors - sectors) << 9;
- bvec_split_segs(q, &bv, &nsegs,
- §ors, max_segs);
+ bv.bv_len =
+ (ctx.max_sectors - ctx.sectors) << 9;
+ bvec_split_segs(q, &bv, &ctx.nsegs,
+ &ctx.sectors, ctx.max_segs);
}
goto split;
}
- if (nsegs == max_segs)
+ if (ctx.nsegs == ctx.max_segs)
goto split;
- bvprv = bv;
- bvprvp = &bvprv;
+ ctx.bvprv = bv;
+ ctx.prv_valid = true;
if (bv.bv_offset + bv.bv_len <= PAGE_SIZE) {
- nsegs++;
- sectors += bv.bv_len >> 9;
- } else if (bvec_split_segs(q, &bv, &nsegs, §ors,
- max_segs)) {
+ ctx.nsegs++;
+ ctx.sectors += bv.bv_len >> 9;
+ } else if (bvec_split_segs(q, &bv, &ctx.nsegs, &ctx.sectors,
+ ctx.max_segs)) {
goto split;
}
}
do_split = false;
split:
- *segs = nsegs;
+ *segs = ctx.nsegs;
if (do_split) {
- new = bio_split(bio, sectors, GFP_NOIO, bs);
+ new = bio_split(bio, ctx.sectors, GFP_NOIO, bs);
if (new)
bio = new;
}
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 10/28] block: Create generic vec_split_segs() from bvec_split_segs()
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
bvec_split_segs() only requires the address and length of the
vector. In order to generalize it to work with dma_vecs, we just
take the address and length directly instead of the bio_vec.
The function is renamed to vec_split_segs() and a helper is added
to avoid having to adjust the existing callsites.
Note: the new bvec_split_segs() helper will be removed in a subsequent
patch.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
block/blk-merge.c | 21 ++++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 17713d7d98d5..3581c7ac3c1b 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -158,13 +158,13 @@ static unsigned get_max_segment_size(struct request_queue *q,
}
/*
- * Split the bvec @bv into segments, and update all kinds of
- * variables.
+ * Split the an address/offset and length into segments, and
+ * update all kinds of variables.
*/
-static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv,
- unsigned *nsegs, unsigned *sectors, unsigned max_segs)
+static bool vec_split_segs(struct request_queue *q, unsigned offset,
+ unsigned len, unsigned *nsegs, unsigned *sectors,
+ unsigned max_segs)
{
- unsigned len = bv->bv_len;
unsigned total_len = 0;
unsigned new_nsegs = 0, seg_size = 0;
@@ -173,14 +173,14 @@ static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv,
* current bvec has to be splitted as multiple segments.
*/
while (len && new_nsegs + *nsegs < max_segs) {
- seg_size = get_max_segment_size(q, bv->bv_offset + total_len);
+ seg_size = get_max_segment_size(q, offset + total_len);
seg_size = min(seg_size, len);
new_nsegs++;
total_len += seg_size;
len -= seg_size;
- if ((bv->bv_offset + total_len) & queue_virt_boundary(q))
+ if ((offset + total_len) & queue_virt_boundary(q))
break;
}
@@ -194,6 +194,13 @@ static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv,
return !!len;
}
+static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv,
+ unsigned *nsegs, unsigned *sectors, unsigned max_segs)
+{
+ return vec_split_segs(q, bv->bv_offset, bv->bv_len, nsegs,
+ sectors, max_segs);
+}
+
static struct bio *blk_bio_segment_split(struct request_queue *q,
struct bio *bio,
struct bio_set *bs,
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 09/28] block: Introduce vec_gap_to_prev()
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
Introduce vec_gap_to_prev() which is a more general
form of bvec_gap_to_prev().
In order to support splitting dma_vecs we will need to do a similar
calcualtion using the DMA address and length.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
block/blk.h | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/block/blk.h b/block/blk.h
index 4142383eed7a..c5512fefe703 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -98,11 +98,19 @@ static inline bool dmavec_phys_mergeable(struct request_queue *q,
vec2->dv_addr, vec2->dv_len);
}
+static inline bool __vec_gap_to_prev(struct request_queue *q,
+ unsigned int prv_offset, unsigned int prv_len,
+ unsigned int nxt_offset)
+{
+ return (nxt_offset & queue_virt_boundary(q)) ||
+ ((prv_offset + prv_len) & queue_virt_boundary(q));
+}
+
static inline bool __bvec_gap_to_prev(struct request_queue *q,
struct bio_vec *bprv, unsigned int offset)
{
- return (offset & queue_virt_boundary(q)) ||
- ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
+ return __vec_gap_to_prev(q, bprv->bv_offset, bprv->bv_len,
+ offset);
}
/*
@@ -117,6 +125,15 @@ static inline bool bvec_gap_to_prev(struct request_queue *q,
return __bvec_gap_to_prev(q, bprv, offset);
}
+static inline bool vec_gap_to_prev(struct request_queue *q,
+ unsigned int prv_offset, unsigned int prv_len,
+ unsigned int nxt_offset)
+{
+ if (!queue_virt_boundary(q))
+ return false;
+ return __vec_gap_to_prev(q, prv_offset, prv_len, nxt_offset);
+}
+
#ifdef CONFIG_BLK_DEV_INTEGRITY
void blk_flush_integrity(void);
bool __bio_integrity_endio(struct bio *);
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 08/28] block: Introduce dmavec_phys_mergeable()
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
Introduce a new helper which is an analog of biovec_phys_mergeable()
for dma-direct vectors.
This also provides a common helper vec_phys_mergeable() for use in
code that's general to both bio_vecs and dma_vecs.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
block/blk.h | 28 ++++++++++++++++++++++------
1 file changed, 22 insertions(+), 6 deletions(-)
diff --git a/block/blk.h b/block/blk.h
index 7814aa207153..4142383eed7a 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -66,20 +66,36 @@ static inline void blk_queue_enter_live(struct request_queue *q)
percpu_ref_get(&q->q_usage_counter);
}
+static inline bool vec_phys_mergeable(struct request_queue *q,
+ unsigned long addr1, unsigned int len1,
+ unsigned long addr2, unsigned int len2)
+{
+ unsigned long mask = queue_segment_boundary(q);
+
+ if (addr1 + len1 != addr2)
+ return false;
+ if ((addr1 | mask) != ((addr2 + len2 - 1) | mask))
+ return false;
+ return true;
+}
+
static inline bool biovec_phys_mergeable(struct request_queue *q,
struct bio_vec *vec1, struct bio_vec *vec2)
{
- unsigned long mask = queue_segment_boundary(q);
phys_addr_t addr1 = page_to_phys(vec1->bv_page) + vec1->bv_offset;
phys_addr_t addr2 = page_to_phys(vec2->bv_page) + vec2->bv_offset;
- if (addr1 + vec1->bv_len != addr2)
- return false;
if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page))
return false;
- if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask))
- return false;
- return true;
+
+ return vec_phys_mergeable(q, addr1, vec1->bv_len, addr2, vec2->bv_len);
+}
+
+static inline bool dmavec_phys_mergeable(struct request_queue *q,
+ struct dma_vec *vec1, struct dma_vec *vec2)
+{
+ return vec_phys_mergeable(q, vec1->dv_addr, vec1->dv_len,
+ vec2->dv_addr, vec2->dv_len);
}
static inline bool __bvec_gap_to_prev(struct request_queue *q,
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 07/28] block: Use dma_vec length in bio_cur_bytes() for dma-direct bios
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
For dma-direct bios, use the dv_len of the current vector
seeing the bio_vec's are not valid in such a context.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
include/linux/bio.h | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/include/linux/bio.h b/include/linux/bio.h
index e212e5958a75..df7973932525 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -91,10 +91,12 @@ static inline bool bio_mergeable(struct bio *bio)
static inline unsigned int bio_cur_bytes(struct bio *bio)
{
- if (bio_has_data(bio))
- return bio_iovec(bio).bv_len;
- else /* dataless requests such as discard */
+ if (!bio_has_data(bio)) /* dataless requests such as discard */
return bio->bi_iter.bi_size;
+ else if (op_is_dma_direct(bio->bi_opf))
+ return bio_dma_vec(bio).dv_len;
+ else
+ return bio_iovec(bio).bv_len;
}
static inline void *bio_data(struct bio *bio)
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 06/28] block: Support dma-direct bios in bio_advance_iter()
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
Dma-direct bio iterators need to be advanced using a similar
dvec_iter_advance helper.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
include/linux/bio.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 8180309123d7..e212e5958a75 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -134,6 +134,8 @@ static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
if (bio_no_advance_iter(bio))
iter->bi_size -= bytes;
+ else if (op_is_dma_direct(bio->bi_opf))
+ dvec_iter_advance(bio->bi_dma_vec, iter, bytes);
else
bvec_iter_advance(bio->bi_io_vec, iter, bytes);
/* TODO: It is reasonable to complete bio with error here. */
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 05/28] block: Skip dma-direct bios in bio_integrity_prep()
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
The block layer will not be able to handle integrity for dma-direct
bios seeing it does not have access to the underlying data.
If users of dma-direct require integrity, they will have to handle it
in the layer creating the bios. This is left as future work should
somebody care about handling such a case.
Thus, bio_integrity_prep() should ignore dma-direct bios.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
block/bio-integrity.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 4db620849515..10fdf456fcd8 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -221,6 +221,10 @@ bool bio_integrity_prep(struct bio *bio)
if (bio_integrity(bio))
return true;
+ /* The block layer cannot handle integrity for dma-direct bios */
+ if (bio_is_dma_direct(bio))
+ return true;
+
if (bio_data_dir(bio) == READ) {
if (!bi->profile->verify_fn ||
!(bi->flags & BLK_INTEGRITY_VERIFY))
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 04/28] block: Never bounce dma-direct bios
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
It is expected the creator of the dma-direct bio will ensure the
target device can access the DMA address it's creating bios for.
It's also not possible to bounce a dma-direct bio seeing the block
layer doesn't have any way to access the underlying data behind
the DMA address.
Thus, never bounce dma-direct bios.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
block/bounce.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/block/bounce.c b/block/bounce.c
index f8ed677a1bf7..17e020a40cca 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -367,6 +367,14 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig)
if (!bio_has_data(*bio_orig))
return;
+ /*
+ * For DMA direct bios, Upper layers are expected to ensure
+ * the device in question can access the DMA addresses. So
+ * it never makes sense to bounce a DMA direct bio.
+ */
+ if (bio_is_dma_direct(*bio_orig))
+ return;
+
/*
* for non-isa bounce case, just check if the bounce pfn is equal
* to or bigger than the highest pfn in the system -- in that case,
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 03/28] block: Warn on mis-use of dma-direct bios
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
This is a result of an audit of users of 'bi_io_vec'. A number of
warnings and blocking conditions are added to ensure dma-direct bios
are not incorrectly accessing the 'bi_io_vec' when they should access
the 'bi_dma_vec'. These are largely just protecting against mis-uses
in future development so depending on taste and public opinion some
or all of these checks may not be necessary.
A few other issues with dma-direct bios will be tackled in subsequent
patches.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
block/bio.c | 33 +++++++++++++++++++++++++++++++++
block/blk-core.c | 3 +++
2 files changed, 36 insertions(+)
diff --git a/block/bio.c b/block/bio.c
index 683cbb40f051..6998fceddd36 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -525,6 +525,9 @@ void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
struct bio_vec bv;
struct bvec_iter iter;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return;
+
__bio_for_each_segment(bv, bio, iter, start) {
char *data = bvec_kmap_irq(&bv, &flags);
memset(data, 0, bv.bv_len);
@@ -707,6 +710,8 @@ static int __bio_add_pc_page(struct request_queue *q, struct bio *bio,
*/
if (unlikely(bio_flagged(bio, BIO_CLONED)))
return 0;
+ if (unlikely(bio_is_dma_direct(bio)))
+ return 0;
if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q))
return 0;
@@ -783,6 +788,8 @@ bool __bio_try_merge_page(struct bio *bio, struct page *page,
{
if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
return false;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return false;
if (bio->bi_vcnt > 0) {
struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
@@ -814,6 +821,7 @@ void __bio_add_page(struct bio *bio, struct page *page,
WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
WARN_ON_ONCE(bio_full(bio));
+ WARN_ON_ONCE(bio_is_dma_direct(bio));
bv->bv_page = page;
bv->bv_offset = off;
@@ -851,6 +859,8 @@ static void bio_get_pages(struct bio *bio)
struct bvec_iter_all iter_all;
struct bio_vec *bvec;
+ WARN_ON_ONCE(bio_is_dma_direct(bio));
+
bio_for_each_segment_all(bvec, bio, iter_all)
get_page(bvec->bv_page);
}
@@ -956,6 +966,8 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
if (WARN_ON_ONCE(bio->bi_vcnt))
return -EINVAL;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return -EINVAL;
do {
if (is_bvec)
@@ -1029,6 +1041,9 @@ void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
void *src_p, *dst_p;
unsigned bytes;
+ if (WARN_ON_ONCE(bio_is_dma_direct(src) || bio_is_dma_direct(dst)))
+ return;
+
while (src_iter->bi_size && dst_iter->bi_size) {
src_bv = bio_iter_iovec(src, *src_iter);
dst_bv = bio_iter_iovec(dst, *dst_iter);
@@ -1143,6 +1158,9 @@ static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
struct bio_vec *bvec;
struct bvec_iter_all iter_all;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return -EINVAL;
+
bio_for_each_segment_all(bvec, bio, iter_all) {
ssize_t ret;
@@ -1174,6 +1192,9 @@ static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
struct bio_vec *bvec;
struct bvec_iter_all iter_all;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return -EINVAL;
+
bio_for_each_segment_all(bvec, bio, iter_all) {
ssize_t ret;
@@ -1197,6 +1218,9 @@ void bio_free_pages(struct bio *bio)
struct bio_vec *bvec;
struct bvec_iter_all iter_all;
+ if (WARN_ON_ONCE(bio_is_dma_direct(bio)))
+ return;
+
bio_for_each_segment_all(bvec, bio, iter_all)
__free_page(bvec->bv_page);
}
@@ -1653,6 +1677,9 @@ void bio_set_pages_dirty(struct bio *bio)
struct bio_vec *bvec;
struct bvec_iter_all iter_all;
+ if (unlikely(bio_is_dma_direct(bio)))
+ return;
+
bio_for_each_segment_all(bvec, bio, iter_all) {
if (!PageCompound(bvec->bv_page))
set_page_dirty_lock(bvec->bv_page);
@@ -1704,6 +1731,9 @@ void bio_check_pages_dirty(struct bio *bio)
unsigned long flags;
struct bvec_iter_all iter_all;
+ if (unlikely(bio_is_dma_direct(bio)))
+ return;
+
bio_for_each_segment_all(bvec, bio, iter_all) {
if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page))
goto defer;
@@ -1777,6 +1807,9 @@ void bio_flush_dcache_pages(struct bio *bi)
struct bio_vec bvec;
struct bvec_iter iter;
+ if (unlikely(bio_is_dma_direct(bi)))
+ return;
+
bio_for_each_segment(bvec, bi, iter)
flush_dcache_page(bvec.bv_page);
}
diff --git a/block/blk-core.c b/block/blk-core.c
index 8340f69670d8..ea152d54c7ce 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1467,6 +1467,9 @@ void rq_flush_dcache_pages(struct request *rq)
struct req_iterator iter;
struct bio_vec bvec;
+ if (unlikely(blk_rq_is_dma_direct(rq)))
+ return;
+
rq_for_each_segment(bvec, rq, iter)
flush_dcache_page(bvec.bv_page);
}
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 02/28] block: Add dma_vec structure
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
The dma_vec structure is similar to the bio_vec structure except
it only stores DMA addresses instead of the struct page address.
struct bios will be able to make use of dma_vecs with a union and,
therefore, we need to ensure that struct dma_vec is no larger
than struct bvec, as they will share the allocated memory.
dma_vecs can make the same use of the bvec_iter structure
to iterate through the vectors.
This will be used for passing DMA addresses directly through the block
layer. I expect something like struct dma_vec will also be used in
Christoph's work to improve the dma_mapping layer and remove sgls.
At some point, these would use the same structure.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
include/linux/bio.h | 12 +++++++++++
include/linux/blk_types.h | 5 ++++-
include/linux/bvec.h | 43 +++++++++++++++++++++++++++++++++++++++
3 files changed, 59 insertions(+), 1 deletion(-)
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 0f23b5682640..8180309123d7 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -28,6 +28,8 @@
#define bio_iter_iovec(bio, iter) \
bvec_iter_bvec((bio)->bi_io_vec, (iter))
+#define bio_iter_dma_vec(bio, iter) \
+ bvec_iter_dvec((bio)->bi_dma_vec, (iter))
#define bio_iter_page(bio, iter) \
bvec_iter_page((bio)->bi_io_vec, (iter))
@@ -39,6 +41,7 @@
#define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter)
#define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter)
#define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter)
+#define bio_dma_vec(bio) bio_iter_dma_vec((bio), (bio)->bi_iter)
#define bio_multiple_segments(bio) \
((bio)->bi_iter.bi_size != bio_iovec(bio).bv_len)
@@ -155,6 +158,15 @@ static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
#define bio_for_each_bvec(bvl, bio, iter) \
__bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
+#define __bio_for_each_dvec(dvl, bio, iter, start) \
+ for (iter = (start); \
+ (iter).bi_size && \
+ ((dvl = bvec_iter_dvec((bio)->bi_dma_vec, (iter))), 1); \
+ dvec_iter_advance((bio)->bi_dma_vec, &(iter), (dvl).dv_len))
+
+#define bio_for_each_dvec(dvl, bio, iter) \
+ __bio_for_each_dvec(dvl, bio, iter, (bio)->bi_iter)
+
#define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
static inline unsigned bio_segments(struct bio *bio)
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index f3cabfdb6774..7f76ea73b77d 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -191,7 +191,10 @@ struct bio {
atomic_t __bi_cnt; /* pin count */
- struct bio_vec *bi_io_vec; /* the actual vec list */
+ union {
+ struct bio_vec *bi_io_vec; /* the actual vec list */
+ struct dma_vec *bi_dma_vec; /* for dma direct bios*/
+ };
struct bio_set *bi_pool;
diff --git a/include/linux/bvec.h b/include/linux/bvec.h
index a032f01e928c..f680e96132ef 100644
--- a/include/linux/bvec.h
+++ b/include/linux/bvec.h
@@ -21,6 +21,11 @@ struct bio_vec {
unsigned int bv_offset;
};
+struct dma_vec {
+ dma_addr_t dv_addr;
+ unsigned int dv_len;
+};
+
struct bvec_iter {
sector_t bi_sector; /* device address in 512 byte
sectors */
@@ -84,6 +89,18 @@ struct bvec_iter_all {
.bv_offset = bvec_iter_offset((bvec), (iter)), \
})
+#define bvec_iter_dvec_addr(dvec, iter) \
+ (__bvec_iter_bvec((dvec), (iter))->dv_addr + (iter).bi_bvec_done)
+#define bvec_iter_dvec_len(dvec, iter) \
+ min((iter).bi_size, \
+ __bvec_iter_bvec((dvec), (iter))->dv_len - (iter).bi_bvec_done)
+
+#define bvec_iter_dvec(dvec, iter) \
+((struct dma_vec) { \
+ .dv_addr = bvec_iter_dvec_addr((dvec), (iter)), \
+ .dv_len = bvec_iter_dvec_len((dvec), (iter)), \
+})
+
static inline bool bvec_iter_advance(const struct bio_vec *bv,
struct bvec_iter *iter, unsigned bytes)
{
@@ -110,6 +127,32 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv,
return true;
}
+static inline bool dvec_iter_advance(const struct dma_vec *dv,
+ struct bvec_iter *iter, unsigned bytes)
+{
+ if (WARN_ONCE(bytes > iter->bi_size,
+ "Attempted to advance past end of dvec iter\n")) {
+ iter->bi_size = 0;
+ return false;
+ }
+
+ while (bytes) {
+ const struct dma_vec *cur = dv + iter->bi_idx;
+ unsigned len = min3(bytes, iter->bi_size,
+ cur->dv_len - iter->bi_bvec_done);
+
+ bytes -= len;
+ iter->bi_size -= len;
+ iter->bi_bvec_done += len;
+
+ if (iter->bi_bvec_done == cur->dv_len) {
+ iter->bi_bvec_done = 0;
+ iter->bi_idx++;
+ }
+ }
+ return true;
+}
+
#define for_each_bvec(bvl, bio_vec, iter, start) \
for (iter = (start); \
(iter).bi_size && \
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 01/28] block: Introduce DMA direct request type
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
In-Reply-To: <20190620161240.22738-1-logang@deltatee.com>
A DMA direct request allows passing DMA addresses directly through
the block layer, instead of struct pages. This allows the calling
layer to take care of the mapping and unmapping and also creates
a path to doing peer-to-peer transactions without using struct pages.
Signed-off-by: Logan Gunthorpe <logang@deltatee.com>
---
include/linux/blk_types.h | 9 ++++++++-
include/linux/blkdev.h | 10 ++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 95202f80676c..f3cabfdb6774 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -322,6 +322,7 @@ enum req_flag_bits {
__REQ_NOUNMAP, /* do not free blocks when zeroing */
__REQ_HIPRI,
+ __REQ_DMA_DIRECT, /* DMA address direct request */
/* for driver use */
__REQ_DRV,
@@ -345,6 +346,7 @@ enum req_flag_bits {
#define REQ_NOWAIT (1ULL << __REQ_NOWAIT)
#define REQ_NOUNMAP (1ULL << __REQ_NOUNMAP)
#define REQ_HIPRI (1ULL << __REQ_HIPRI)
+#define REQ_DMA_DIRECT (1ULL << __REQ_DMA_DIRECT)
#define REQ_DRV (1ULL << __REQ_DRV)
#define REQ_SWAP (1ULL << __REQ_SWAP)
@@ -353,7 +355,7 @@ enum req_flag_bits {
(REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER)
#define REQ_NOMERGE_FLAGS \
- (REQ_NOMERGE | REQ_PREFLUSH | REQ_FUA)
+ (REQ_NOMERGE | REQ_PREFLUSH | REQ_FUA | REQ_DMA_DIRECT)
enum stat_group {
STAT_READ,
@@ -412,6 +414,11 @@ static inline int op_stat_group(unsigned int op)
return op_is_write(op);
}
+static inline int op_is_dma_direct(unsigned int op)
+{
+ return op & REQ_DMA_DIRECT;
+}
+
typedef unsigned int blk_qc_t;
#define BLK_QC_T_NONE -1U
#define BLK_QC_T_SHIFT 16
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 592669bcc536..ce70d5dded5f 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -271,6 +271,16 @@ static inline bool bio_is_passthrough(struct bio *bio)
return blk_op_is_scsi(op) || blk_op_is_private(op);
}
+static inline bool bio_is_dma_direct(struct bio *bio)
+{
+ return op_is_dma_direct(bio->bi_opf);
+}
+
+static inline bool blk_rq_is_dma_direct(struct request *rq)
+{
+ return op_is_dma_direct(rq->cmd_flags);
+}
+
static inline unsigned short req_get_ioprio(struct request *req)
{
return req->ioprio;
--
2.20.1
^ permalink raw reply related
* [RFC PATCH 00/28] Removing struct page from P2PDMA
From: Logan Gunthorpe @ 2019-06-20 16:12 UTC (permalink / raw)
To: linux-kernel, linux-block, linux-nvme, linux-pci, linux-rdma
Cc: Jens Axboe, Christoph Hellwig, Bjorn Helgaas, Dan Williams,
Sagi Grimberg, Keith Busch, Jason Gunthorpe, Stephen Bates,
Logan Gunthorpe
For eons there has been a debate over whether or not to use
struct pages for peer-to-peer DMA transactions. Pro-pagers have
argued that struct pages are necessary for interacting with
existing code like scatterlists or the bio_vecs. Anti-pagers
assert that the tracking of the memory is unecessary and
allocating the pages is a waste of memory. Both viewpoints are
valid, however developers working on GPUs and RDMA tend to be
able to do away with struct pages relatively easily compared to
those wanting to work with NVMe devices through the block layer.
So it would be of great value to be able to universally do P2PDMA
transactions without the use of struct pages.
Previously, there have been multiple attempts[1][2] to replace
struct page usage with pfn_t but this has been unpopular seeing
it creates dangerous edge cases where unsuspecting code might
run accross pfn_t's they are not ready for.
Currently, we have P2PDMA using struct pages through the block layer
and the dangerous cases are avoided by using a queue flag that
indicates support for the special pages.
This RFC proposes a new solution: allow the block layer to take
DMA addresses directly for queues that indicate support. This will
provide a more general path for doing P2PDMA-like requests and will
allow us to remove the struct pages that back P2PDMA memory thus paving
the way to build a more uniform P2PDMA ecosystem.
This is a fairly long patch set but most of the patches are quite
small. Patches 1 through 18 introduce the concept of a dma_vec that
is similar to a bio_vec (except it takes dma_addr_t's instead of pages
and offsets) as well as a special dma-direct bio/request. Most of these
patches just prevent the new type of bio from being mis-used and
also support splitting and mapping them in the same way that struct
page bios can be operated on. Patches 19 through 22 modify the existing
P2PDMA support in nvme-pci, ib-core and nvmet to use DMA addresses
directly. Patches 23 through 25 remove the P2PDMA specific
code from the block layer and ib-core. Finally, patches 26 through 28
remove the struct pages from the PCI P2PDMA code.
This RFC is based on v5.2-rc5 and a git branch is available here:
https://github.com/sbates130272/linux-p2pmem.git dma_direct_rfc1
[1] https://lwn.net/Articles/647404/
[2] https://lore.kernel.org/lkml/1495662147-18277-1-git-send-email-logang@deltatee.com/
--
Logan Gunthorpe (28):
block: Introduce DMA direct request type
block: Add dma_vec structure
block: Warn on mis-use of dma-direct bios
block: Never bounce dma-direct bios
block: Skip dma-direct bios in bio_integrity_prep()
block: Support dma-direct bios in bio_advance_iter()
block: Use dma_vec length in bio_cur_bytes() for dma-direct bios
block: Introduce dmavec_phys_mergeable()
block: Introduce vec_gap_to_prev()
block: Create generic vec_split_segs() from bvec_split_segs()
block: Create blk_segment_split_ctx
block: Create helper for bvec_should_split()
block: Generalize bvec_should_split()
block: Support splitting dma-direct bios
block: Support counting dma-direct bio segments
block: Implement mapping dma-direct requests to SGs in blk_rq_map_sg()
block: Introduce queue flag to indicate support for dma-direct bios
block: Introduce bio_add_dma_addr()
nvme-pci: Support dma-direct bios
IB/core: Introduce API for initializing a RW ctx from a DMA address
nvmet: Split nvmet_bdev_execute_rw() into a helper function
nvmet: Use DMA addresses instead of struct pages for P2P
nvme-pci: Remove support for PCI_P2PDMA requests
block: Remove PCI_P2PDMA queue flag
IB/core: Remove P2PDMA mapping support in rdma_rw_ctx
PCI/P2PDMA: Remove SGL helpers
PCI/P2PDMA: Remove struct pages that back P2PDMA memory
memremap: Remove PCI P2PDMA page memory type
Documentation/driver-api/pci/p2pdma.rst | 9 +-
block/bio-integrity.c | 4 +
block/bio.c | 71 +++++++
block/blk-core.c | 3 +
block/blk-merge.c | 256 ++++++++++++++++++------
block/blk.h | 49 ++++-
block/bounce.c | 8 +
drivers/infiniband/core/rw.c | 85 ++++++--
drivers/nvme/host/core.c | 4 +-
drivers/nvme/host/nvme.h | 2 +-
drivers/nvme/host/pci.c | 29 ++-
drivers/nvme/target/core.c | 12 +-
drivers/nvme/target/io-cmd-bdev.c | 82 +++++---
drivers/nvme/target/nvmet.h | 5 +-
drivers/nvme/target/rdma.c | 43 +++-
drivers/pci/p2pdma.c | 202 +++----------------
include/linux/bio.h | 32 ++-
include/linux/blk_types.h | 14 +-
include/linux/blkdev.h | 16 +-
include/linux/bvec.h | 43 ++++
include/linux/memremap.h | 5 -
include/linux/mm.h | 13 --
include/linux/pci-p2pdma.h | 19 --
include/rdma/rw.h | 6 +
24 files changed, 648 insertions(+), 364 deletions(-)
--
2.20.1
^ permalink raw reply
* [PATCH v4 01/25] sysfs: export sysfs_remove_file_self()
From: Jack Wang @ 2019-06-20 15:03 UTC (permalink / raw)
To: linux-block, linux-rdma
Cc: axboe, hch, sagi, bvanassche, jgg, dledford, danil.kipnis,
rpenyaev, Roman Pen, linux-kernel
In-Reply-To: <20190620150337.7847-1-jinpuwang@gmail.com>
From: Roman Pen <roman.penyaev@profitbricks.com>
Function is going to be used in transport over RDMA module
in subsequent patches.
Signed-off-by: Roman Pen <roman.penyaev@profitbricks.com>
Acked-by: Tejun Heo <tj@kernel.org>
Cc: linux-kernel@vger.kernel.org
---
fs/sysfs/file.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/fs/sysfs/file.c b/fs/sysfs/file.c
index 130fc6fbcc03..1ff4672d7746 100644
--- a/fs/sysfs/file.c
+++ b/fs/sysfs/file.c
@@ -492,6 +492,7 @@ bool sysfs_remove_file_self(struct kobject *kobj, const struct attribute *attr)
kernfs_put(kn);
return ret;
}
+EXPORT_SYMBOL_GPL(sysfs_remove_file_self);
void sysfs_remove_files(struct kobject *kobj, const struct attribute * const *ptr)
{
--
2.17.1
^ permalink raw reply related
* Re: [PATCH][next] RDMA: check for null return from call to ib_get_client_data
From: Jason Gunthorpe @ 2019-06-20 15:02 UTC (permalink / raw)
To: Colin King
Cc: Doug Ledford, Leon Romanovsky, Parav Pandit, linux-rdma,
kernel-janitors, linux-kernel
In-Reply-To: <20190620135052.27367-1-colin.king@canonical.com>
On Thu, Jun 20, 2019 at 02:50:52PM +0100, Colin King wrote:
> From: Colin Ian King <colin.king@canonical.com>
>
> The return from ib_get_client_data can potentially be null, so add a null
> check on umad_dev and return -ENODEV in this unlikely case to avoid any
> null pointer deferences.
It would be a kernel bug if NULL is seen here.
Jason
^ permalink raw reply
* Re: [PATCH RFC 00/10] RDMA/FS DAX truncate proposal
From: Jan Kara @ 2019-06-20 14:52 UTC (permalink / raw)
To: Matthew Wilcox
Cc: Jason Gunthorpe, Theodore Ts'o,
linux-nvdimm-hn68Rpc1hR1g9hUCZPvPmw,
linux-rdma-u79uwXL29TY76Z2rM5mHXA, Jeff Layton, Dave Chinner,
linux-kernel-u79uwXL29TY76Z2rM5mHXA,
linux-xfs-u79uwXL29TY76Z2rM5mHXA, linux-mm-Bw31MaZKKs3YtjvyW6yDsg,
Jérôme Glisse, John Hubbard,
linux-fsdevel-u79uwXL29TY76Z2rM5mHXA, Jan Kara,
linux-ext4-u79uwXL29TY76Z2rM5mHXA, Andrew Morton
In-Reply-To: <20190613152755.GI32656-PfSpb0PWhxZc2C7mugBRk2EX/6BAtgUQ@public.gmane.org>
On Thu 13-06-19 08:27:55, Matthew Wilcox wrote:
> On Thu, Jun 13, 2019 at 10:25:55AM +1000, Dave Chinner wrote:
> > e.g. Process A has an exclusive layout lease on file F. It does an
> > IO to file F. The filesystem IO path checks that Process A owns the
> > lease on the file and so skips straight through layout breaking
> > because it owns the lease and is allowed to modify the layout. It
> > then takes the inode metadata locks to allocate new space and write
> > new data.
> >
> > Process B now tries to write to file F. The FS checks whether
> > Process B owns a layout lease on file F. It doesn't, so then it
> > tries to break the layout lease so the IO can proceed. The layout
> > breaking code sees that process A has an exclusive layout lease
> > granted, and so returns -ETXTBSY to process B - it is not allowed to
> > break the lease and so the IO fails with -ETXTBSY.
>
> This description doesn't match the behaviour that RDMA wants either.
> Even if Process A has a lease on the file, an IO from Process A which
> results in blocks being freed from the file is going to result in the
> RDMA device being able to write to blocks which are now freed (and
> potentially reallocated to another file).
I think you're partially wrong here. You are correct that the lease won't
stop process A from doing truncate on the file. *But* there are still page
pins in existence so truncate will block on waiting for these pins to go
away (after all this is a protection that guards all short-term page pin
users). So there is no problem with blocks being freed under the RDMA app.
Yes, the app will effectively deadlock and sysadmin has to kill it. IMO an
acceptable answer for doing something stupid and unsupportable...
Honza
--
Jan Kara <jack-IBi9RG/b67k@public.gmane.org>
SUSE Labs, CR
^ permalink raw reply
* [PATCH][next] RDMA: check for null return from call to ib_get_client_data
From: Colin King @ 2019-06-20 13:50 UTC (permalink / raw)
To: Doug Ledford, Jason Gunthorpe, Leon Romanovsky, Parav Pandit,
linux-rdma
Cc: kernel-janitors, linux-kernel
From: Colin Ian King <colin.king@canonical.com>
The return from ib_get_client_data can potentially be null, so add a null
check on umad_dev and return -ENODEV in this unlikely case to avoid any
null pointer deferences.
Addresses-Coverity: ("Dereference null return")
Fixes: 8f71bb0030b8 ("RDMA: Report available cdevs through RDMA_NLDEV_CMD_GET_CHARDEV")
Signed-off-by: Colin Ian King <colin.king@canonical.com>
---
drivers/infiniband/core/user_mad.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index 547090b41cfb..d78a35913824 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -1153,6 +1153,9 @@ static int ib_issm_get_nl_info(struct ib_device *ibdev, void *client_data,
struct ib_umad_device *umad_dev =
ib_get_client_data(ibdev, &umad_client);
+ if (!umad_dev)
+ return -ENODEV;
+
if (!rdma_is_port_valid(ibdev, res->port))
return -EINVAL;
--
2.20.1
^ permalink raw reply related
* Re: use exact allocation for dma coherent memory
From: Christoph Hellwig @ 2019-06-20 10:51 UTC (permalink / raw)
To: Jason Gunthorpe
Cc: Christoph Hellwig, Potnuri Bharat Teja, Dan Carpenter,
Maarten Lankhorst, Maxime Ripard, Sean Paul, David Airlie,
Daniel Vetter, Jani Nikula, Joonas Lahtinen, Rodrigo Vivi,
Ian Abbott, H Hartley Sweeten, devel, linux-s390,
Intel Linux Wireless, linux-rdma, netdev, intel-gfx,
linux-wireless, linux-kernel
In-Reply-To: <20190619162903.GF9360@ziepe.ca>
On Wed, Jun 19, 2019 at 01:29:03PM -0300, Jason Gunthorpe wrote:
> > Yes. This will blow up badly on many platforms, as sq->queue
> > might be vmapped, ioremapped, come from a pool without page backing.
>
> Gah, this addr gets fed into io_remap_pfn_range/remap_pfn_range too..
>
> Potnuri, you should fix this..
>
> You probably need to use dma_mmap_from_dev_coherent() in the mmap ?
The function to use is dma_mmap_coherent, dma_mmap_from_dev_coherent is
just an internal helper.
That beiŋ said the drivers/infiniband code has a lot of
*remap_pfn_range, and a lot of them look like they might be for
DMA memory.
^ permalink raw reply
* [PATCH iproute2-next v2 2/2] uapi: update if_link.h
From: Denis Kirjanov @ 2019-06-20 9:02 UTC (permalink / raw)
To: stephen; +Cc: netdev, linux-rdma, dledford, mkubecek, Denis Kirjanov
In-Reply-To: <20190620090249.106704-1-dkirjanov@suse.com>
update if_link.h to commit 75345f888f700c4ab2448287e35d48c760b202e6
("ipoib: show VF broadcast address")
Signed-off-by: Denis Kirjanov <kda@linux-powerpc.org>
---
include/uapi/linux/if_link.h | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index bfe7f9e6..5f271d84 100644
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -692,6 +692,7 @@ enum {
IFLA_VF_IB_NODE_GUID, /* VF Infiniband node GUID */
IFLA_VF_IB_PORT_GUID, /* VF Infiniband port GUID */
IFLA_VF_VLAN_LIST, /* nested list of vlans, option for QinQ */
+ IFLA_VF_BROADCAST, /* VF broadcast */
__IFLA_VF_MAX,
};
@@ -702,6 +703,10 @@ struct ifla_vf_mac {
__u8 mac[32]; /* MAX_ADDR_LEN */
};
+struct ifla_vf_broadcast {
+ __u8 broadcast[32];
+};
+
struct ifla_vf_vlan {
__u32 vf;
__u32 vlan; /* 0 - 4095, 0 disables VLAN filter */
--
2.12.3
^ permalink raw reply related
* [PATCH iproute2-next v2 1/2] ipaddress: correctly print a VF hw address in the IPoIB case
From: Denis Kirjanov @ 2019-06-20 9:02 UTC (permalink / raw)
To: stephen; +Cc: netdev, linux-rdma, dledford, mkubecek, Denis Kirjanov
Current code assumes that we print Etheret mac and
that doesn't work in IPoIB case with SRIOV-enabled hardware
Before:
11: ib1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 2044 qdisc pfifo_fast
state UP mode DEFAULT group default qlen 256
link/infiniband
80:00:00:66:fe:80:00:00:00:00:00:00:24:8a:07:03:00:a4:3e:7c brd
00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff
vf 0 MAC 14:80:00:00:66:fe, spoof checking off, link-state disable,
trust off, query_rss off
...
After:
11: ib1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 2044 qdisc pfifo_fast
state UP mode DEFAULT group default qlen 256
link/infiniband
80:00:00:66:fe:80:00:00:00:00:00:00:24:8a:07:03:00:a4:3e:7c brd
00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff
vf 0 link/infiniband
80:00:00:66:fe:80:00:00:00:00:00:00:24:8a:07:03:00:a4:3e:7c brd
00:ff:ff:ff:ff:12:40:1b:ff:ff:00:00:00:00:00:00:ff:ff:ff:ff, spoof
checking off, link-state disable, trust off, query_rss off
v1->v2: updated kernel headers to uapi commit
Signed-off-by: Denis Kirjanov <kda@linux-powerpc.org>
---
ip/ipaddress.c | 42 +++++++++++++++++++++++++++++++++++++-----
1 file changed, 37 insertions(+), 5 deletions(-)
diff --git a/ip/ipaddress.c b/ip/ipaddress.c
index b504200b..13ad76dd 100644
--- a/ip/ipaddress.c
+++ b/ip/ipaddress.c
@@ -26,6 +26,7 @@
#include <linux/netdevice.h>
#include <linux/if_arp.h>
+#include <linux/if_infiniband.h>
#include <linux/sockios.h>
#include <linux/net_namespace.h>
@@ -349,9 +350,10 @@ static void print_af_spec(FILE *fp, struct rtattr *af_spec_attr)
static void print_vf_stats64(FILE *fp, struct rtattr *vfstats);
-static void print_vfinfo(FILE *fp, struct rtattr *vfinfo)
+static void print_vfinfo(struct ifinfomsg *ifi, FILE *fp, struct rtattr *vfinfo)
{
struct ifla_vf_mac *vf_mac;
+ struct ifla_vf_broadcast *vf_broadcast;
struct ifla_vf_tx_rate *vf_tx_rate;
struct rtattr *vf[IFLA_VF_MAX + 1] = {};
@@ -365,13 +367,43 @@ static void print_vfinfo(FILE *fp, struct rtattr *vfinfo)
parse_rtattr_nested(vf, IFLA_VF_MAX, vfinfo);
vf_mac = RTA_DATA(vf[IFLA_VF_MAC]);
+ vf_broadcast = RTA_DATA(vf[IFLA_VF_BROADCAST]);
vf_tx_rate = RTA_DATA(vf[IFLA_VF_TX_RATE]);
print_string(PRINT_FP, NULL, "%s ", _SL_);
print_int(PRINT_ANY, "vf", "vf %d ", vf_mac->vf);
- print_string(PRINT_ANY, "mac", "MAC %s",
- ll_addr_n2a((unsigned char *) &vf_mac->mac,
- ETH_ALEN, 0, b1, sizeof(b1)));
+
+ print_string(PRINT_ANY,
+ "link_type",
+ " link/%s ",
+ ll_type_n2a(ifi->ifi_type, b1, sizeof(b1)));
+
+ print_color_string(PRINT_ANY,
+ COLOR_MAC,
+ "address",
+ "%s",
+ ll_addr_n2a((unsigned char *) &vf_mac->mac,
+ ifi->ifi_type == ARPHRD_ETHER ? ETH_ALEN : INFINIBAND_ALEN,
+ ifi->ifi_type,
+ b1, sizeof(b1)));
+
+ if (vf[IFLA_VF_BROADCAST]) {
+ if (ifi->ifi_flags&IFF_POINTOPOINT) {
+ print_string(PRINT_FP, NULL, " peer ", NULL);
+ print_bool(PRINT_JSON,
+ "link_pointtopoint", NULL, true);
+ } else {
+ print_string(PRINT_FP, NULL, " brd ", NULL);
+ }
+ print_color_string(PRINT_ANY,
+ COLOR_MAC,
+ "broadcast",
+ "%s",
+ ll_addr_n2a((unsigned char *) &vf_broadcast->broadcast,
+ ifi->ifi_type == ARPHRD_ETHER ? ETH_ALEN : INFINIBAND_ALEN,
+ ifi->ifi_type,
+ b1, sizeof(b1)));
+ }
if (vf[IFLA_VF_VLAN_LIST]) {
struct rtattr *i, *vfvlanlist = vf[IFLA_VF_VLAN_LIST];
@@ -1102,7 +1134,7 @@ int print_linkinfo(struct nlmsghdr *n, void *arg)
open_json_array(PRINT_JSON, "vfinfo_list");
for (i = RTA_DATA(vflist); RTA_OK(i, rem); i = RTA_NEXT(i, rem)) {
open_json_object(NULL);
- print_vfinfo(fp, i);
+ print_vfinfo(ifi, fp, i);
close_json_object();
}
close_json_array(PRINT_JSON, NULL);
--
2.12.3
^ permalink raw reply related
* Re: [PATCH 7/8] mpt3sas: set an unlimited max_segment_size for SAS 3.0 HBAs
From: Suganath Prabu Subramani @ 2019-06-20 7:08 UTC (permalink / raw)
To: Ming Lei
Cc: Christoph Hellwig, Martin K . Petersen, Sagi Grimberg,
Max Gurtovoy, Bart Van Assche, linux-rdma, Linux SCSI List,
megaraidlinux.pdl, PDL-MPT-FUSIONLINUX, linux-hyperv,
Linux Kernel Mailing List
In-Reply-To: <CA+RiK64sFfY79i7q2YbN5HcZ4wzVOcLWgDJnPbf6=ycdcmC-Mg@mail.gmail.com>
Acked-by: Suganath Prabu <suganath-prabu.subramani@broadcom.com>
On Thu, Jun 20, 2019 at 12:34 PM Suganath Prabu Subramani
<suganath-prabu.subramani@broadcom.com> wrote:
>
> Please consider this as Acked-by: Suganath Prabu
> <suganath-prabu.subramani@broadcom.com>
>
>
> On Tue, Jun 18, 2019 at 6:16 AM Ming Lei <tom.leiming@gmail.com> wrote:
> >
> > On Mon, Jun 17, 2019 at 8:21 PM Christoph Hellwig <hch@lst.de> wrote:
> > >
> > > When using a virt_boundary_mask, as done for NVMe devices attached to
> > > mpt3sas controllers we require an unlimited max_segment_size, as the
> > > virt boundary merging code assumes that. But we also need to propagate
> > > that to the DMA mapping layer to make dma-debug happy. The SCSI layer
> > > takes care of that when using the per-host virt_boundary setting, but
> > > given that mpt3sas only wants to set the virt_boundary for actual
> > > NVMe devices we can't rely on that. The DMA layer maximum segment
> > > is global to the HBA however, so we have to set it explicitly. This
> > > patch assumes that mpt3sas does not have a segment size limitation,
> > > which seems true based on the SGL format, but will need to be verified.
> > >
> > > Signed-off-by: Christoph Hellwig <hch@lst.de>
> > > ---
> > > drivers/scsi/mpt3sas/mpt3sas_scsih.c | 1 +
> > > 1 file changed, 1 insertion(+)
> > >
> > > diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
> > > index 1ccfbc7eebe0..c719b807f6d8 100644
> > > --- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
> > > +++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
> > > @@ -10222,6 +10222,7 @@ static struct scsi_host_template mpt3sas_driver_template = {
> > > .this_id = -1,
> > > .sg_tablesize = MPT3SAS_SG_DEPTH,
> > > .max_sectors = 32767,
> > > + .max_segment_size = 0xffffffff,
> >
> > .max_segment_size should be aligned, either setting it here correctly or
> > forcing to make it aligned in scsi-core.
> >
> > Thanks,
> > Ming Lei
^ permalink raw reply
* Re: [PATCH 7/8] mpt3sas: set an unlimited max_segment_size for SAS 3.0 HBAs
From: Suganath Prabu Subramani @ 2019-06-20 7:04 UTC (permalink / raw)
To: Ming Lei
Cc: Christoph Hellwig, Martin K . Petersen, Sagi Grimberg,
Max Gurtovoy, Bart Van Assche, linux-rdma, Linux SCSI List,
megaraidlinux.pdl, PDL-MPT-FUSIONLINUX, linux-hyperv,
Linux Kernel Mailing List
In-Reply-To: <CACVXFVObpdjN6V9qS-C9NG5xcrPqmx-X22qVamOSZf81Vog6zw@mail.gmail.com>
Please consider this as Acked-by: Suganath Prabu
<suganath-prabu.subramani@broadcom.com>
On Tue, Jun 18, 2019 at 6:16 AM Ming Lei <tom.leiming@gmail.com> wrote:
>
> On Mon, Jun 17, 2019 at 8:21 PM Christoph Hellwig <hch@lst.de> wrote:
> >
> > When using a virt_boundary_mask, as done for NVMe devices attached to
> > mpt3sas controllers we require an unlimited max_segment_size, as the
> > virt boundary merging code assumes that. But we also need to propagate
> > that to the DMA mapping layer to make dma-debug happy. The SCSI layer
> > takes care of that when using the per-host virt_boundary setting, but
> > given that mpt3sas only wants to set the virt_boundary for actual
> > NVMe devices we can't rely on that. The DMA layer maximum segment
> > is global to the HBA however, so we have to set it explicitly. This
> > patch assumes that mpt3sas does not have a segment size limitation,
> > which seems true based on the SGL format, but will need to be verified.
> >
> > Signed-off-by: Christoph Hellwig <hch@lst.de>
> > ---
> > drivers/scsi/mpt3sas/mpt3sas_scsih.c | 1 +
> > 1 file changed, 1 insertion(+)
> >
> > diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
> > index 1ccfbc7eebe0..c719b807f6d8 100644
> > --- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c
> > +++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c
> > @@ -10222,6 +10222,7 @@ static struct scsi_host_template mpt3sas_driver_template = {
> > .this_id = -1,
> > .sg_tablesize = MPT3SAS_SG_DEPTH,
> > .max_sectors = 32767,
> > + .max_segment_size = 0xffffffff,
>
> .max_segment_size should be aligned, either setting it here correctly or
> forcing to make it aligned in scsi-core.
>
> Thanks,
> Ming Lei
^ permalink raw reply
* [PATCH mlx5-next] net/mlx5: Convert mkey_table to XArray
From: Saeed Mahameed @ 2019-06-20 7:03 UTC (permalink / raw)
To: Saeed Mahameed, Leon Romanovsky
Cc: Jason Gunthorpe, netdev@vger.kernel.org,
linux-rdma@vger.kernel.org, Matthew Wilcox
From: Matthew Wilcox <willy@infradead.org>
The lock protecting the data structure does not need to be an rwlock. The
only read access to the lock is in an error path, and if that's limiting
your scalability, you have bigger performance problems.
Eliminate mlx5_mkey_table in favour of using the xarray directly.
reg_mr_callback must use GFP_ATOMIC for allocating XArray nodes as it may
be called in interrupt context.
This also fixes a minor bug where SRCU locking was being used on the radix
tree read side, when RCU was needed too.
Change-Id: Ifeedbbc8f2c856cc044094d03167aae9f9162400
Signed-off-by: Matthew Wilcox <willy@infradead.org>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
---
drivers/infiniband/hw/mlx5/cq.c | 8 +++---
drivers/infiniband/hw/mlx5/devx.c | 18 +++----------
drivers/infiniband/hw/mlx5/mr.c | 10 ++++----
drivers/infiniband/hw/mlx5/odp.c | 10 ++++----
drivers/net/ethernet/mellanox/mlx5/core/mr.c | 27 ++++++++------------
include/linux/mlx5/driver.h | 13 ++--------
include/linux/mlx5/qp.h | 5 ----
7 files changed, 31 insertions(+), 60 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index 2e2e65f00257..0220736b073e 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -522,9 +522,9 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,
case MLX5_CQE_SIG_ERR:
sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64;
- read_lock(&dev->mdev->priv.mkey_table.lock);
- mmkey = __mlx5_mr_lookup(dev->mdev,
- mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey)));
+ xa_lock(&dev->mdev->priv.mkey_table);
+ mmkey = xa_load(&dev->mdev->priv.mkey_table,
+ mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey)));
mr = to_mibmr(mmkey);
get_sig_err_item(sig_err_cqe, &mr->sig->err_item);
mr->sig->sig_err_exists = true;
@@ -537,7 +537,7 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,
mr->sig->err_item.expected,
mr->sig->err_item.actual);
- read_unlock(&dev->mdev->priv.mkey_table.lock);
+ xa_unlock(&dev->mdev->priv.mkey_table);
goto repoll;
}
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 80b42d069328..931f587dfb8f 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -1043,13 +1043,10 @@ static int devx_handle_mkey_indirect(struct devx_obj *obj,
struct mlx5_ib_dev *dev,
void *in, void *out)
{
- struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table;
struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr;
- unsigned long flags;
struct mlx5_core_mkey *mkey;
void *mkc;
u8 key;
- int err;
mkey = &devx_mr->mmkey;
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
@@ -1062,11 +1059,8 @@ static int devx_handle_mkey_indirect(struct devx_obj *obj,
mkey->pd = MLX5_GET(mkc, mkc, pd);
devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
- write_lock_irqsave(&table->lock, flags);
- err = radix_tree_insert(&table->tree, mlx5_base_mkey(mkey->key),
- mkey);
- write_unlock_irqrestore(&table->lock, flags);
- return err;
+ return xa_err(xa_store(&dev->mdev->priv.mkey_table,
+ mlx5_base_mkey(mkey->key), mkey, GFP_KERNEL));
}
static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
@@ -1117,12 +1111,8 @@ static void devx_free_indirect_mkey(struct rcu_head *rcu)
*/
static void devx_cleanup_mkey(struct devx_obj *obj)
{
- struct mlx5_mkey_table *table = &obj->mdev->priv.mkey_table;
- unsigned long flags;
-
- write_lock_irqsave(&table->lock, flags);
- radix_tree_delete(&table->tree, mlx5_base_mkey(obj->devx_mr.mmkey.key));
- write_unlock_irqrestore(&table->lock, flags);
+ xa_erase(&obj->mdev->priv.mkey_table,
+ mlx5_base_mkey(obj->devx_mr.mmkey.key));
}
static int devx_obj_cleanup(struct ib_uobject *uobject,
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 5f09699fab98..83b452d977d4 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -130,7 +130,7 @@ static void reg_mr_callback(int status, struct mlx5_async_work *context)
struct mlx5_cache_ent *ent = &cache->ent[c];
u8 key;
unsigned long flags;
- struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table;
+ struct xarray *mkeys = &dev->mdev->priv.mkey_table;
int err;
spin_lock_irqsave(&ent->lock, flags);
@@ -158,12 +158,12 @@ static void reg_mr_callback(int status, struct mlx5_async_work *context)
ent->size++;
spin_unlock_irqrestore(&ent->lock, flags);
- write_lock_irqsave(&table->lock, flags);
- err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmkey.key),
- &mr->mmkey);
+ xa_lock_irqsave(mkeys, flags);
+ err = xa_err(__xa_store(mkeys, mlx5_base_mkey(mr->mmkey.key),
+ &mr->mmkey, GFP_ATOMIC));
+ xa_unlock_irqrestore(mkeys, flags);
if (err)
pr_err("Error inserting to mkey tree. 0x%x\n", -err);
- write_unlock_irqrestore(&table->lock, flags);
if (!completion_done(&ent->compl))
complete(&ent->compl);
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 12ccee1eb047..c594489eb2d7 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -768,7 +768,7 @@ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev,
bcnt -= *bytes_committed;
next_mr:
- mmkey = __mlx5_mr_lookup(dev->mdev, mlx5_base_mkey(key));
+ mmkey = xa_load(&dev->mdev->priv.mkey_table, mlx5_base_mkey(key));
if (!mkey_is_eq(mmkey, key)) {
mlx5_ib_dbg(dev, "failed to find mkey %x\n", key);
ret = -EFAULT;
@@ -1686,8 +1686,8 @@ static void num_pending_prefetch_dec(struct mlx5_ib_dev *dev,
struct mlx5_core_mkey *mmkey;
struct mlx5_ib_mr *mr;
- mmkey = __mlx5_mr_lookup(dev->mdev,
- mlx5_base_mkey(sg_list[i].lkey));
+ mmkey = xa_load(&dev->mdev->priv.mkey_table,
+ mlx5_base_mkey(sg_list[i].lkey));
mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);
atomic_dec(&mr->num_pending_prefetch);
}
@@ -1706,8 +1706,8 @@ static bool num_pending_prefetch_inc(struct ib_pd *pd,
struct mlx5_core_mkey *mmkey;
struct mlx5_ib_mr *mr;
- mmkey = __mlx5_mr_lookup(dev->mdev,
- mlx5_base_mkey(sg_list[i].lkey));
+ mmkey = xa_load(&dev->mdev->priv.mkey_table,
+ mlx5_base_mkey(sg_list[i].lkey));
if (!mmkey || mmkey->key != sg_list[i].lkey) {
ret = false;
break;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mr.c b/drivers/net/ethernet/mellanox/mlx5/core/mr.c
index ea744d8466ea..9231b39d18b2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mr.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mr.c
@@ -38,15 +38,12 @@
void mlx5_init_mkey_table(struct mlx5_core_dev *dev)
{
- struct mlx5_mkey_table *table = &dev->priv.mkey_table;
-
- memset(table, 0, sizeof(*table));
- rwlock_init(&table->lock);
- INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
+ xa_init_flags(&dev->priv.mkey_table, XA_FLAGS_LOCK_IRQ);
}
void mlx5_cleanup_mkey_table(struct mlx5_core_dev *dev)
{
+ WARN_ON(!xa_empty(&dev->priv.mkey_table));
}
int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
@@ -56,8 +53,8 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
mlx5_async_cbk_t callback,
struct mlx5_async_work *context)
{
- struct mlx5_mkey_table *table = &dev->priv.mkey_table;
u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {0};
+ struct xarray *mkeys = &dev->priv.mkey_table;
u32 mkey_index;
void *mkc;
int err;
@@ -88,12 +85,10 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
mlx5_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n",
mkey_index, key, mkey->key);
- /* connect to mkey tree */
- write_lock_irq(&table->lock);
- err = radix_tree_insert(&table->tree, mlx5_base_mkey(mkey->key), mkey);
- write_unlock_irq(&table->lock);
+ err = xa_err(xa_store_irq(mkeys, mlx5_base_mkey(mkey->key), mkey,
+ GFP_KERNEL));
if (err) {
- mlx5_core_warn(dev, "failed radix tree insert of mkey 0x%x, %d\n",
+ mlx5_core_warn(dev, "failed xarray insert of mkey 0x%x, %d\n",
mlx5_base_mkey(mkey->key), err);
mlx5_core_destroy_mkey(dev, mkey);
}
@@ -114,17 +109,17 @@ EXPORT_SYMBOL(mlx5_core_create_mkey);
int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev,
struct mlx5_core_mkey *mkey)
{
- struct mlx5_mkey_table *table = &dev->priv.mkey_table;
u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {0};
u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {0};
+ struct xarray *mkeys = &dev->priv.mkey_table;
struct mlx5_core_mkey *deleted_mkey;
unsigned long flags;
- write_lock_irqsave(&table->lock, flags);
- deleted_mkey = radix_tree_delete(&table->tree, mlx5_base_mkey(mkey->key));
- write_unlock_irqrestore(&table->lock, flags);
+ xa_lock_irqsave(mkeys, flags);
+ deleted_mkey = __xa_erase(mkeys, mlx5_base_mkey(mkey->key));
+ xa_unlock_irqrestore(mkeys, flags);
if (!deleted_mkey) {
- mlx5_core_dbg(dev, "failed radix tree delete of mkey 0x%x\n",
+ mlx5_core_dbg(dev, "failed xarray delete of mkey 0x%x\n",
mlx5_base_mkey(mkey->key));
return -ENOENT;
}
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index d8ab633406c2..87f77ded78d4 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -41,7 +41,7 @@
#include <linux/semaphore.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
-#include <linux/radix-tree.h>
+#include <linux/xarray.h>
#include <linux/workqueue.h>
#include <linux/mempool.h>
#include <linux/interrupt.h>
@@ -452,13 +452,6 @@ struct mlx5_qp_table {
struct radix_tree_root tree;
};
-struct mlx5_mkey_table {
- /* protect radix tree
- */
- rwlock_t lock;
- struct radix_tree_root tree;
-};
-
struct mlx5_vf_context {
int enabled;
u64 port_guid;
@@ -546,9 +539,7 @@ struct mlx5_priv {
struct dentry *cmdif_debugfs;
/* end: qp staff */
- /* start: mkey staff */
- struct mlx5_mkey_table mkey_table;
- /* end: mkey staff */
+ struct xarray mkey_table;
/* start: alloc staff */
/* protect buffer alocation according to numa node */
diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h
index 3ba4edbd17a6..d1f353c64797 100644
--- a/include/linux/mlx5/qp.h
+++ b/include/linux/mlx5/qp.h
@@ -551,11 +551,6 @@ static inline struct mlx5_core_qp *__mlx5_qp_lookup(struct mlx5_core_dev *dev, u
return radix_tree_lookup(&dev->priv.qp_table.tree, qpn);
}
-static inline struct mlx5_core_mkey *__mlx5_mr_lookup(struct mlx5_core_dev *dev, u32 key)
-{
- return radix_tree_lookup(&dev->priv.mkey_table.tree, key);
-}
-
int mlx5_core_create_dct(struct mlx5_core_dev *dev,
struct mlx5_core_dct *qp,
u32 *in, int inlen,
--
2.21.0
^ permalink raw reply related
* RE: [PATCH 1/8] scsi: add a host / host template field for the virt boundary
From: Kashyap Desai @ 2019-06-20 6:17 UTC (permalink / raw)
To: Ming Lei, Christoph Hellwig
Cc: Martin K . Petersen, Sagi Grimberg, Max Gurtovoy, Bart Van Assche,
linux-rdma, Linux SCSI List, PDL,MEGARAIDLINUX,
PDL-MPT-FUSIONLINUX, linux-hyperv, Linux Kernel Mailing List,
Kashyap Desai
In-Reply-To: <CACVXFVOwCeM2JzefBpKsVZrEaWpSBR0DF8qp4oKfoHm+pwLBYw@mail.gmail.com>
> -----Original Message-----
> From: megaraidlinux.pdl@broadcom.com
> [mailto:megaraidlinux.pdl@broadcom.com] On Behalf Of Ming Lei
> Sent: Tuesday, June 18, 2019 6:05 AM
> To: Christoph Hellwig <hch@lst.de>
> Cc: Martin K . Petersen <martin.petersen@oracle.com>; Sagi Grimberg
> <sagi@grimberg.me>; Max Gurtovoy <maxg@mellanox.com>; Bart Van
> Assche <bvanassche@acm.org>; linux-rdma <linux-rdma@vger.kernel.org>;
> Linux SCSI List <linux-scsi@vger.kernel.org>;
> megaraidlinux.pdl@broadcom.com; MPT-FusionLinux.pdl@broadcom.com;
> linux-hyperv@vger.kernel.org; Linux Kernel Mailing List <linux-
> kernel@vger.kernel.org>
> Subject: Re: [PATCH 1/8] scsi: add a host / host template field for the
> virt
> boundary
>
> On Mon, Jun 17, 2019 at 8:21 PM Christoph Hellwig <hch@lst.de> wrote:
> >
> > This allows drivers setting it up easily instead of branching out to
> > block layer calls in slave_alloc, and ensures the upgraded
> > max_segment_size setting gets picked up by the DMA layer.
> >
> > Signed-off-by: Christoph Hellwig <hch@lst.de>
> > ---
> > drivers/scsi/hosts.c | 3 +++
> > drivers/scsi/scsi_lib.c | 3 ++-
> > include/scsi/scsi_host.h | 3 +++
> > 3 files changed, 8 insertions(+), 1 deletion(-)
> >
> > diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c index
> > ff0d8c6a8d0c..55522b7162d3 100644
> > --- a/drivers/scsi/hosts.c
> > +++ b/drivers/scsi/hosts.c
> > @@ -462,6 +462,9 @@ struct Scsi_Host *scsi_host_alloc(struct
> scsi_host_template *sht, int privsize)
> > else
> > shost->dma_boundary = 0xffffffff;
> >
> > + if (sht->virt_boundary_mask)
> > + shost->virt_boundary_mask = sht->virt_boundary_mask;
> > +
> > device_initialize(&shost->shost_gendev);
> > dev_set_name(&shost->shost_gendev, "host%d", shost->host_no);
> > shost->shost_gendev.bus = &scsi_bus_type; diff --git
> > a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index
> > 65d0a10c76ad..d333bb6b1c59 100644
> > --- a/drivers/scsi/scsi_lib.c
> > +++ b/drivers/scsi/scsi_lib.c
> > @@ -1775,7 +1775,8 @@ void __scsi_init_queue(struct Scsi_Host *shost,
> struct request_queue *q)
> > dma_set_seg_boundary(dev, shost->dma_boundary);
> >
> > blk_queue_max_segment_size(q, shost->max_segment_size);
> > - dma_set_max_seg_size(dev, shost->max_segment_size);
> > + blk_queue_virt_boundary(q, shost->virt_boundary_mask);
> > + dma_set_max_seg_size(dev, queue_max_segment_size(q));
>
> The patch looks fine, also suggest to make sure that max_segment_size is
> block-size aligned, and un-aligned max segment size has caused trouble on
> mmc.
I validated changes on latest and few older series controllers.
Post changes, I noticed max_segment_size is updated.
find /sys/ -name max_segment_size |grep sdb |xargs grep -r .
/sys/devices/pci0000:3a/0000:3a:00.0/0000:3b:00.0/0000:3c:04.0/0000:40:00.0/0000:41:00.0/0000:42:00.0/host0/target0:2:12/0:2:12:0/block/sdb/queue/max_segment_size:4294967295
I verify that single SGE having 1MB transfer length is working fine.
Acked-by: Kashyap Desai < kashyap.desai@broadcom.com>
>
> Thanks,
> Ming Lei
>
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox