* [PATCH 09/18] io_uring: use fget/fput_many() for file references
From: Jens Axboe @ 2019-02-01 15:24 UTC (permalink / raw)
To: linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh, Jens Axboe
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
Add a separate io_submit_state structure, to cache some of the things
we need for IO submission.
One such example is file reference batching. io_submit_state. We get as
many references as the number of sqes we are submitting, and drop
unused ones if we end up switching files. The assumption here is that
we're usually only dealing with one fd, and if there are multiple,
hopefuly they are at least somewhat ordered. Could trivially be extended
to cover multiple fds, if needed.
On the completion side we do the same thing, except this is trivially
done just locally in io_iopoll_reap().
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 142 ++++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 121 insertions(+), 21 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index b4dd9d2c4b28..400ea8ffbc10 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -139,6 +139,19 @@ struct io_kiocb {
#define IO_PLUG_THRESHOLD 2
#define IO_IOPOLL_BATCH 8
+struct io_submit_state {
+ struct blk_plug plug;
+
+ /*
+ * File reference cache
+ */
+ struct file *file;
+ unsigned int fd;
+ unsigned int has_refs;
+ unsigned int used_refs;
+ unsigned int ios_left;
+};
+
static struct kmem_cache *req_cachep;
static const struct file_operations io_uring_fops;
@@ -286,9 +299,11 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
struct list_head *done)
{
void *reqs[IO_IOPOLL_BATCH];
+ int file_count, to_free;
+ struct file *file = NULL;
struct io_kiocb *req;
- int to_free = 0;
+ file_count = to_free = 0;
while (!list_empty(done)) {
req = list_first_entry(done, struct io_kiocb, list);
list_del(&req->list);
@@ -298,12 +313,28 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
reqs[to_free++] = req;
(*nr_events)++;
- fput(req->rw.ki_filp);
+ /*
+ * Batched puts of the same file, to avoid dirtying the
+ * file usage count multiple times, if avoidable.
+ */
+ if (!file) {
+ file = req->rw.ki_filp;
+ file_count = 1;
+ } else if (file == req->rw.ki_filp) {
+ file_count++;
+ } else {
+ fput_many(file, file_count);
+ file = req->rw.ki_filp;
+ file_count = 1;
+ }
+
if (to_free == ARRAY_SIZE(reqs))
io_free_req_many(ctx, reqs, &to_free);
}
io_commit_cqring(ctx);
+ if (file)
+ fput_many(file, file_count);
io_free_req_many(ctx, reqs, &to_free);
}
@@ -484,8 +515,50 @@ static void io_iopoll_req_issued(struct io_kiocb *req)
list_add_tail(&req->list, &ctx->poll_list);
}
+static void io_file_put(struct io_submit_state *state, struct file *file)
+{
+ if (!state) {
+ fput(file);
+ } else if (state->file) {
+ int diff = state->has_refs - state->used_refs;
+
+ if (diff)
+ fput_many(state->file, diff);
+ state->file = NULL;
+ }
+}
+
+/*
+ * Get as many references to a file as we have IOs left in this submission,
+ * assuming most submissions are for one file, or at least that each file
+ * has more than one submission.
+ */
+static struct file *io_file_get(struct io_submit_state *state, int fd)
+{
+ if (!state)
+ return fget(fd);
+
+ if (state->file) {
+ if (state->fd == fd) {
+ state->used_refs++;
+ state->ios_left--;
+ return state->file;
+ }
+ io_file_put(state, NULL);
+ }
+ state->file = fget_many(fd, state->ios_left);
+ if (!state->file)
+ return NULL;
+
+ state->fd = fd;
+ state->has_refs = state->ios_left;
+ state->used_refs = 1;
+ state->ios_left--;
+ return state->file;
+}
+
static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
- bool force_nonblock)
+ bool force_nonblock, struct io_submit_state *state)
{
struct io_ring_ctx *ctx = req->ctx;
struct kiocb *kiocb = &req->rw;
@@ -493,7 +566,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
int fd, ret;
fd = READ_ONCE(sqe->fd);
- kiocb->ki_filp = fget(fd);
+ kiocb->ki_filp = io_file_get(state, fd);
if (unlikely(!kiocb->ki_filp))
return -EBADF;
kiocb->ki_pos = READ_ONCE(sqe->off);
@@ -535,7 +608,10 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
}
return 0;
out_fput:
- fput(kiocb->ki_filp);
+ /* in case of error, we didn't use this file reference. drop it. */
+ if (state)
+ state->used_refs--;
+ io_file_put(state, kiocb->ki_filp);
return ret;
}
@@ -581,7 +657,7 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
}
static ssize_t io_read(struct io_kiocb *req, const struct sqe_submit *s,
- bool force_nonblock)
+ bool force_nonblock, struct io_submit_state *state)
{
struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
struct kiocb *kiocb = &req->rw;
@@ -589,7 +665,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct sqe_submit *s,
struct file *file;
ssize_t ret;
- ret = io_prep_rw(req, s->sqe, force_nonblock);
+ ret = io_prep_rw(req, s->sqe, force_nonblock, state);
if (ret)
return ret;
file = kiocb->ki_filp;
@@ -624,7 +700,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct sqe_submit *s,
}
static ssize_t io_write(struct io_kiocb *req, const struct sqe_submit *s,
- bool force_nonblock)
+ bool force_nonblock, struct io_submit_state *state)
{
struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
struct kiocb *kiocb = &req->rw;
@@ -632,7 +708,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct sqe_submit *s,
struct file *file;
ssize_t ret;
- ret = io_prep_rw(req, s->sqe, force_nonblock);
+ ret = io_prep_rw(req, s->sqe, force_nonblock, state);
if (ret)
return ret;
file = kiocb->ki_filp;
@@ -732,7 +808,8 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
}
static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
- const struct sqe_submit *s, bool force_nonblock)
+ const struct sqe_submit *s, bool force_nonblock,
+ struct io_submit_state *state)
{
ssize_t ret;
int opcode;
@@ -747,10 +824,10 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
ret = io_nop(req, req->user_data);
break;
case IORING_OP_READV:
- ret = io_read(req, s, force_nonblock);
+ ret = io_read(req, s, force_nonblock, state);
break;
case IORING_OP_WRITEV:
- ret = io_write(req, s, force_nonblock);
+ ret = io_write(req, s, force_nonblock, state);
break;
case IORING_OP_FSYNC:
ret = io_fsync(req, s->sqe, force_nonblock);
@@ -807,7 +884,7 @@ static void io_sq_wq_submit_work(struct work_struct *work)
s->needs_lock = true;
do {
- ret = __io_submit_sqe(ctx, req, s, false);
+ ret = __io_submit_sqe(ctx, req, s, false, NULL);
/*
* We can get EAGAIN for polled IO even though we're forcing
* a sync submission from here, since we can't wait for
@@ -832,7 +909,8 @@ static void io_sq_wq_submit_work(struct work_struct *work)
task_unlock(current);
}
-static int io_submit_sqe(struct io_ring_ctx *ctx, const struct sqe_submit *s)
+static int io_submit_sqe(struct io_ring_ctx *ctx, const struct sqe_submit *s,
+ struct io_submit_state *state)
{
struct io_kiocb *req;
ssize_t ret;
@@ -845,7 +923,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, const struct sqe_submit *s)
if (unlikely(!req))
return -EAGAIN;
- ret = __io_submit_sqe(ctx, req, s, true);
+ ret = __io_submit_sqe(ctx, req, s, true, state);
if (ret == -EAGAIN) {
memcpy(&req->submit, s, sizeof(*s));
INIT_WORK(&req->work, io_sq_wq_submit_work);
@@ -858,6 +936,26 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, const struct sqe_submit *s)
return ret;
}
+/*
+ * Batched submission is done, ensure local IO is flushed out.
+ */
+static void io_submit_state_end(struct io_submit_state *state)
+{
+ blk_finish_plug(&state->plug);
+ io_file_put(state, NULL);
+}
+
+/*
+ * Start submission side cache.
+ */
+static void io_submit_state_start(struct io_submit_state *state,
+ struct io_ring_ctx *ctx, unsigned max_ios)
+{
+ blk_start_plug(&state->plug);
+ state->file = NULL;
+ state->ios_left = max_ios;
+}
+
static void io_commit_sqring(struct io_ring_ctx *ctx)
{
struct io_sq_ring *ring = ctx->sq_ring;
@@ -920,11 +1018,13 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
{
+ struct io_submit_state state, *statep = NULL;
int i, ret = 0, submit = 0;
- struct blk_plug plug;
- if (to_submit > IO_PLUG_THRESHOLD)
- blk_start_plug(&plug);
+ if (to_submit > IO_PLUG_THRESHOLD) {
+ io_submit_state_start(&state, ctx, to_submit);
+ statep = &state;
+ }
for (i = 0; i < to_submit; i++) {
struct sqe_submit s;
@@ -935,7 +1035,7 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
s.has_user = true;
s.needs_lock = false;
- ret = io_submit_sqe(ctx, &s);
+ ret = io_submit_sqe(ctx, &s, statep);
if (ret) {
io_drop_sqring(ctx);
break;
@@ -945,8 +1045,8 @@ static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
}
io_commit_sqring(ctx);
- if (to_submit > IO_PLUG_THRESHOLD)
- blk_finish_plug(&plug);
+ if (statep)
+ io_submit_state_end(statep);
return submit ? submit : ret;
}
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* [PATCH 10/18] io_uring: batch io_kiocb allocation
From: Jens Axboe @ 2019-02-01 15:24 UTC (permalink / raw)
To: linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh, Jens Axboe
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
Similarly to how we use the state->ios_left to know how many references
to get to a file, we can use it to allocate the io_kiocb's we need in
bulk.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 45 ++++++++++++++++++++++++++++++++++++++-------
1 file changed, 38 insertions(+), 7 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 400ea8ffbc10..c30b416fc5ea 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -142,6 +142,13 @@ struct io_kiocb {
struct io_submit_state {
struct blk_plug plug;
+ /*
+ * io_kiocb alloc cache
+ */
+ void *reqs[IO_IOPOLL_BATCH];
+ unsigned int free_reqs;
+ unsigned int cur_req;
+
/*
* File reference cache
*/
@@ -259,20 +266,40 @@ static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs)
wake_up(&ctx->wait);
}
-static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx)
+static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
+ struct io_submit_state *state)
{
struct io_kiocb *req;
if (!percpu_ref_tryget(&ctx->refs))
return NULL;
- req = kmem_cache_alloc(req_cachep, __GFP_NOWARN);
- if (req) {
- req->ctx = ctx;
- req->flags = 0;
- return req;
+ if (!state) {
+ req = kmem_cache_alloc(req_cachep, __GFP_NOWARN);
+ if (unlikely(!req))
+ goto out;
+ } else if (!state->free_reqs) {
+ size_t sz;
+ int ret;
+
+ sz = min_t(size_t, state->ios_left, ARRAY_SIZE(state->reqs));
+ ret = kmem_cache_alloc_bulk(req_cachep, __GFP_NOWARN, sz,
+ state->reqs);
+ if (unlikely(ret <= 0))
+ goto out;
+ state->free_reqs = ret - 1;
+ state->cur_req = 1;
+ req = state->reqs[0];
+ } else {
+ req = state->reqs[state->cur_req];
+ state->free_reqs--;
+ state->cur_req++;
}
+ req->ctx = ctx;
+ req->flags = 0;
+ return req;
+out:
io_ring_drop_ctx_refs(ctx, 1);
return NULL;
}
@@ -919,7 +946,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, const struct sqe_submit *s,
if (unlikely(s->sqe->flags))
return -EINVAL;
- req = io_get_req(ctx);
+ req = io_get_req(ctx, state);
if (unlikely(!req))
return -EAGAIN;
@@ -943,6 +970,9 @@ static void io_submit_state_end(struct io_submit_state *state)
{
blk_finish_plug(&state->plug);
io_file_put(state, NULL);
+ if (state->free_reqs)
+ kmem_cache_free_bulk(req_cachep, state->free_reqs,
+ &state->reqs[state->cur_req]);
}
/*
@@ -952,6 +982,7 @@ static void io_submit_state_start(struct io_submit_state *state,
struct io_ring_ctx *ctx, unsigned max_ios)
{
blk_start_plug(&state->plug);
+ state->free_reqs = 0;
state->file = NULL;
state->ios_left = max_ios;
}
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* [PATCH 11/18] block: implement bio helper to add iter bvec pages to bio
From: Jens Axboe @ 2019-02-01 15:24 UTC (permalink / raw)
To: linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh, Jens Axboe
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
For an ITER_BVEC, we can just iterate the iov and add the pages
to the bio directly. This requires that the caller doesn't releases
the pages on IO completion, we add a BIO_NO_PAGE_REF flag for that.
The current two callers of bio_iov_iter_get_pages() are updated to
check if they need to release pages on completion. This makes them
work with bvecs that contain kernel mapped pages already.
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
block/bio.c | 59 ++++++++++++++++++++++++++++++++-------
fs/block_dev.c | 5 ++--
fs/iomap.c | 5 ++--
include/linux/blk_types.h | 1 +
4 files changed, 56 insertions(+), 14 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index 4db1008309ed..330df572cfb8 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -828,6 +828,23 @@ int bio_add_page(struct bio *bio, struct page *page,
}
EXPORT_SYMBOL(bio_add_page);
+static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter)
+{
+ const struct bio_vec *bv = iter->bvec;
+ unsigned int len;
+ size_t size;
+
+ len = min_t(size_t, bv->bv_len, iter->count);
+ size = bio_add_page(bio, bv->bv_page, len,
+ bv->bv_offset + iter->iov_offset);
+ if (size == len) {
+ iov_iter_advance(iter, size);
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
#define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *))
/**
@@ -876,23 +893,43 @@ static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
}
/**
- * bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
+ * bio_iov_iter_get_pages - add user or kernel pages to a bio
* @bio: bio to add pages to
- * @iter: iov iterator describing the region to be mapped
+ * @iter: iov iterator describing the region to be added
+ *
+ * This takes either an iterator pointing to user memory, or one pointing to
+ * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
+ * map them into the kernel. On IO completion, the caller should put those
+ * pages. If we're adding kernel pages, we just have to add the pages to the
+ * bio directly. We don't grab an extra reference to those pages (the user
+ * should already have that), and we don't put the page on IO completion.
+ * The caller needs to check if the bio is flagged BIO_NO_PAGE_REF on IO
+ * completion. If it isn't, then pages should be released.
*
- * Pins pages from *iter and appends them to @bio's bvec array. The
- * pages will have to be released using put_page() when done.
* The function tries, but does not guarantee, to pin as many pages as
- * fit into the bio, or are requested in *iter, whatever is smaller.
- * If MM encounters an error pinning the requested pages, it stops.
- * Error is returned only if 0 pages could be pinned.
+ * fit into the bio, or are requested in *iter, whatever is smaller. If
+ * MM encounters an error pinning the requested pages, it stops. Error
+ * is returned only if 0 pages could be pinned.
*/
int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
{
+ const bool is_bvec = iov_iter_is_bvec(iter);
unsigned short orig_vcnt = bio->bi_vcnt;
+ /*
+ * If this is a BVEC iter, then the pages are kernel pages. Don't
+ * release them on IO completion.
+ */
+ if (is_bvec)
+ bio_set_flag(bio, BIO_NO_PAGE_REF);
+
do {
- int ret = __bio_iov_iter_get_pages(bio, iter);
+ int ret;
+
+ if (is_bvec)
+ ret = __bio_iov_bvec_add_pages(bio, iter);
+ else
+ ret = __bio_iov_iter_get_pages(bio, iter);
if (unlikely(ret))
return bio->bi_vcnt > orig_vcnt ? 0 : ret;
@@ -1634,7 +1671,8 @@ static void bio_dirty_fn(struct work_struct *work)
next = bio->bi_private;
bio_set_pages_dirty(bio);
- bio_release_pages(bio);
+ if (!bio_flagged(bio, BIO_NO_PAGE_REF))
+ bio_release_pages(bio);
bio_put(bio);
}
}
@@ -1650,7 +1688,8 @@ void bio_check_pages_dirty(struct bio *bio)
goto defer;
}
- bio_release_pages(bio);
+ if (!bio_flagged(bio, BIO_NO_PAGE_REF))
+ bio_release_pages(bio);
bio_put(bio);
return;
defer:
diff --git a/fs/block_dev.c b/fs/block_dev.c
index 392e2bfb636f..051ab41d1c61 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -338,8 +338,9 @@ static void blkdev_bio_end_io(struct bio *bio)
struct bio_vec *bvec;
int i;
- bio_for_each_segment_all(bvec, bio, i)
- put_page(bvec->bv_page);
+ if (!bio_flagged(bio, BIO_NO_PAGE_REF))
+ bio_for_each_segment_all(bvec, bio, i)
+ put_page(bvec->bv_page);
bio_put(bio);
}
}
diff --git a/fs/iomap.c b/fs/iomap.c
index 4ee50b76b4a1..e5c48a0b20e0 100644
--- a/fs/iomap.c
+++ b/fs/iomap.c
@@ -1582,8 +1582,9 @@ static void iomap_dio_bio_end_io(struct bio *bio)
struct bio_vec *bvec;
int i;
- bio_for_each_segment_all(bvec, bio, i)
- put_page(bvec->bv_page);
+ if (!bio_flagged(bio, BIO_NO_PAGE_REF))
+ bio_for_each_segment_all(bvec, bio, i)
+ put_page(bvec->bv_page);
bio_put(bio);
}
}
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index d66bf5f32610..791fee35df88 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -215,6 +215,7 @@ struct bio {
/*
* bio flags
*/
+#define BIO_NO_PAGE_REF 0 /* don't put release vec pages */
#define BIO_SEG_VALID 1 /* bi_phys_segments valid */
#define BIO_CLONED 2 /* doesn't own data */
#define BIO_BOUNCED 3 /* bio is a bounce bio */
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* [PATCH 12/18] io_uring: add support for pre-mapped user IO buffers
From: Jens Axboe @ 2019-02-01 15:24 UTC (permalink / raw)
To: linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh, Jens Axboe
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
If we have fixed user buffers, we can map them into the kernel when we
setup the io_context. That avoids the need to do get_user_pages() for
each and every IO.
To utilize this feature, the application must call io_uring_register()
after having setup an io_uring context, passing in
IORING_REGISTER_BUFFERS as the opcode. The argument must be a pointer
to an iovec array, and the nr_args should contain how many iovecs the
application wishes to map.
If successful, these buffers are now mapped into the kernel, eligible
for IO. To use these fixed buffers, the application must use the
IORING_OP_READ_FIXED and IORING_OP_WRITE_FIXED opcodes, and then
set sqe->index to the desired buffer index. sqe->addr..sqe->addr+seq->len
must point to somewhere inside the indexed buffer.
The application may register buffers throughout the lifetime of the
io_uring context. It can call io_uring_register() with
IORING_UNREGISTER_BUFFERS as the opcode to unregister the current set of
buffers, and then register a new set. The application need not
unregister buffers explicitly before shutting down the io_uring context.
It's perfectly valid to setup a larger buffer, and then sometimes only
use parts of it for an IO. As long as the range is within the originally
mapped region, it will work just fine.
For now, buffers must not be file backed. If file backed buffers are
passed in, the registration will fail with -1/EOPNOTSUPP. This
restriction may be relaxed in the future.
RLIMIT_MEMLOCK is used to check how much memory we can pin. A somewhat
arbitrary 1G per buffer size is also imposed.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
arch/x86/entry/syscalls/syscall_32.tbl | 1 +
arch/x86/entry/syscalls/syscall_64.tbl | 1 +
fs/io_uring.c | 345 ++++++++++++++++++++++++-
include/linux/sched/user.h | 2 +-
include/linux/syscalls.h | 2 +
include/uapi/asm-generic/unistd.h | 4 +-
include/uapi/linux/io_uring.h | 13 +-
kernel/sys_ni.c | 1 +
8 files changed, 352 insertions(+), 17 deletions(-)
diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl
index 481c126259e9..2eefd2a7c1ce 100644
--- a/arch/x86/entry/syscalls/syscall_32.tbl
+++ b/arch/x86/entry/syscalls/syscall_32.tbl
@@ -400,3 +400,4 @@
386 i386 rseq sys_rseq __ia32_sys_rseq
425 i386 io_uring_setup sys_io_uring_setup __ia32_sys_io_uring_setup
426 i386 io_uring_enter sys_io_uring_enter __ia32_sys_io_uring_enter
+427 i386 io_uring_register sys_io_uring_register __ia32_sys_io_uring_register
diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl
index 6a32a430c8e0..65c026185e61 100644
--- a/arch/x86/entry/syscalls/syscall_64.tbl
+++ b/arch/x86/entry/syscalls/syscall_64.tbl
@@ -345,6 +345,7 @@
334 common rseq __x64_sys_rseq
425 common io_uring_setup __x64_sys_io_uring_setup
426 common io_uring_enter __x64_sys_io_uring_enter
+427 common io_uring_register __x64_sys_io_uring_register
#
# x32-specific system call numbers start at 512 to avoid cache impact
diff --git a/fs/io_uring.c b/fs/io_uring.c
index c30b416fc5ea..80c788d8c22a 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -25,10 +25,12 @@
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/blkdev.h>
+#include <linux/bvec.h>
#include <linux/anon_inodes.h>
#include <linux/sched/mm.h>
#include <linux/uaccess.h>
#include <linux/nospec.h>
+#include <linux/sizes.h>
#include <uapi/linux/io_uring.h>
@@ -58,6 +60,13 @@ struct io_cq_ring {
struct io_uring_cqe cqes[];
};
+struct io_mapped_ubuf {
+ u64 ubuf;
+ size_t len;
+ struct bio_vec *bvec;
+ unsigned int nr_bvecs;
+};
+
struct io_ring_ctx {
struct {
struct percpu_ref refs;
@@ -91,6 +100,10 @@ struct io_ring_ctx {
struct fasync_struct *cq_fasync;
} ____cacheline_aligned_in_smp;
+ /* if used, fixed mapped user buffers */
+ unsigned nr_user_bufs;
+ struct io_mapped_ubuf *user_bufs;
+
struct user_struct *user;
struct completion ctx_done;
@@ -663,6 +676,44 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
}
}
+static int io_import_fixed(struct io_ring_ctx *ctx, int rw,
+ const struct io_uring_sqe *sqe,
+ struct iov_iter *iter)
+{
+ size_t len = READ_ONCE(sqe->len);
+ struct io_mapped_ubuf *imu;
+ unsigned index, buf_index;
+ size_t offset;
+ u64 buf_addr;
+
+ /* attempt to use fixed buffers without having provided iovecs */
+ if (unlikely(!ctx->user_bufs))
+ return -EFAULT;
+
+ buf_index = READ_ONCE(sqe->buf_index);
+ if (unlikely(buf_index >= ctx->nr_user_bufs))
+ return -EFAULT;
+
+ index = array_index_nospec(buf_index, ctx->nr_user_bufs);
+ imu = &ctx->user_bufs[index];
+ buf_addr = READ_ONCE(sqe->addr);
+
+ if (buf_addr + len < buf_addr)
+ return -EFAULT;
+ if (buf_addr < imu->ubuf || buf_addr + len > imu->ubuf + imu->len)
+ return -EFAULT;
+
+ /*
+ * May not be a start of buffer, set size appropriately
+ * and advance us to the beginning.
+ */
+ offset = buf_addr - imu->ubuf;
+ iov_iter_bvec(iter, rw, imu->bvec, imu->nr_bvecs, offset + len);
+ if (offset)
+ iov_iter_advance(iter, offset);
+ return 0;
+}
+
static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
const struct sqe_submit *s, struct iovec **iovec,
struct iov_iter *iter)
@@ -670,6 +721,15 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
const struct io_uring_sqe *sqe = s->sqe;
void __user *buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
size_t sqe_len = READ_ONCE(sqe->len);
+ u8 opcode;
+
+ opcode = READ_ONCE(sqe->opcode);
+ if (opcode == IORING_OP_READ_FIXED ||
+ opcode == IORING_OP_WRITE_FIXED) {
+ ssize_t ret = io_import_fixed(ctx, rw, sqe, iter);
+ *iovec = NULL;
+ return ret;
+ }
if (!s->has_user)
return EFAULT;
@@ -813,7 +873,7 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
return -EINVAL;
- if (unlikely(sqe->addr || sqe->ioprio))
+ if (unlikely(sqe->addr || sqe->ioprio || sqe->buf_index))
return -EINVAL;
fsync_flags = READ_ONCE(sqe->fsync_flags);
@@ -851,9 +911,19 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
ret = io_nop(req, req->user_data);
break;
case IORING_OP_READV:
+ if (unlikely(s->sqe->buf_index))
+ return -EINVAL;
ret = io_read(req, s, force_nonblock, state);
break;
case IORING_OP_WRITEV:
+ if (unlikely(s->sqe->buf_index))
+ return -EINVAL;
+ ret = io_write(req, s, force_nonblock, state);
+ break;
+ case IORING_OP_READ_FIXED:
+ ret = io_read(req, s, force_nonblock, state);
+ break;
+ case IORING_OP_WRITE_FIXED:
ret = io_write(req, s, force_nonblock, state);
break;
case IORING_OP_FSYNC:
@@ -882,14 +952,23 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
return 0;
}
+static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
+{
+ u8 opcode = READ_ONCE(sqe->opcode);
+
+ return !(opcode == IORING_OP_READ_FIXED ||
+ opcode == IORING_OP_WRITE_FIXED);
+}
+
static void io_sq_wq_submit_work(struct work_struct *work)
{
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
struct sqe_submit *s = &req->submit;
- u64 user_data = READ_ONCE(s->sqe->user_data);
struct io_ring_ctx *ctx = req->ctx;
- mm_segment_t old_fs = get_fs();
struct files_struct *old_files;
+ mm_segment_t old_fs;
+ bool needs_user;
+ u64 user_data;
int ret;
/* Ensure we clear previously set forced non-block flag */
@@ -900,15 +979,25 @@ static void io_sq_wq_submit_work(struct work_struct *work)
current->files = ctx->sqo_files;
task_unlock(current);
- if (!mmget_not_zero(ctx->sqo_mm)) {
- ret = -EFAULT;
- goto err;
- }
-
- use_mm(ctx->sqo_mm);
- set_fs(USER_DS);
- s->has_user = true;
+ user_data = READ_ONCE(s->sqe->user_data);
s->needs_lock = true;
+ s->has_user = false;
+
+ /*
+ * If we're doing IO to fixed buffers, we don't need to get/set
+ * user context
+ */
+ needs_user = io_sqe_needs_user(s->sqe);
+ if (needs_user) {
+ if (!mmget_not_zero(ctx->sqo_mm)) {
+ ret = -EFAULT;
+ goto err;
+ }
+ use_mm(ctx->sqo_mm);
+ old_fs = get_fs();
+ set_fs(USER_DS);
+ s->has_user = true;
+ }
do {
ret = __io_submit_sqe(ctx, req, s, false, NULL);
@@ -922,9 +1011,11 @@ static void io_sq_wq_submit_work(struct work_struct *work)
cond_resched();
} while (1);
- set_fs(old_fs);
- unuse_mm(ctx->sqo_mm);
- mmput(ctx->sqo_mm);
+ if (needs_user) {
+ set_fs(old_fs);
+ unuse_mm(ctx->sqo_mm);
+ mmput(ctx->sqo_mm);
+ }
err:
if (ret) {
io_cqring_add_event(ctx, user_data, ret, 0);
@@ -1205,6 +1296,14 @@ static void *io_mem_alloc(size_t size)
return (void *) __get_free_pages(gfp_flags, get_order(size));
}
+static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
+{
+ if (ctx->user)
+ return __io_account_mem(ctx->user, nr_pages);
+
+ return 0;
+}
+
static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
{
struct io_sq_ring *sq_ring;
@@ -1218,6 +1317,169 @@ static unsigned long ring_pages(unsigned sq_entries, unsigned cq_entries)
return (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
}
+static int io_sqe_buffer_unregister(struct io_ring_ctx *ctx)
+{
+ int i, j;
+
+ if (!ctx->user_bufs)
+ return -ENXIO;
+
+ for (i = 0; i < ctx->sq_entries; i++) {
+ struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
+
+ for (j = 0; j < imu->nr_bvecs; j++)
+ put_page(imu->bvec[j].bv_page);
+
+ io_unaccount_mem(ctx, imu->nr_bvecs);
+ kfree(imu->bvec);
+ imu->nr_bvecs = 0;
+ }
+
+ kfree(ctx->user_bufs);
+ ctx->user_bufs = NULL;
+ free_uid(ctx->user);
+ ctx->user = NULL;
+ return 0;
+}
+
+static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
+ void __user *arg, unsigned index)
+{
+ struct iovec __user *src;
+
+#ifdef CONFIG_COMPAT
+ if (ctx->compat) {
+ struct compat_iovec __user *ciovs;
+ struct compat_iovec ciov;
+
+ ciovs = (struct compat_iovec __user *) arg;
+ if (copy_from_user(&ciov, &ciovs[index], sizeof(ciov)))
+ return -EFAULT;
+
+ dst->iov_base = (void __user *) (unsigned long) ciov.iov_base;
+ dst->iov_len = ciov.iov_len;
+ return 0;
+ }
+#endif
+ src = (struct iovec __user *) arg;
+ if (copy_from_user(dst, &src[index], sizeof(*dst)))
+ return -EFAULT;
+ return 0;
+}
+
+static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
+ unsigned nr_args)
+{
+ struct page **pages = NULL;
+ int i, j, got_pages = 0;
+ int ret = -EINVAL;
+
+ if (ctx->user_bufs)
+ return -EBUSY;
+ if (!nr_args || nr_args > UIO_MAXIOV)
+ return -EINVAL;
+
+ ctx->user_bufs = kcalloc(nr_args, sizeof(struct io_mapped_ubuf),
+ GFP_KERNEL);
+ if (!ctx->user_bufs)
+ return -ENOMEM;
+
+ if (!capable(CAP_IPC_LOCK))
+ ctx->user = get_uid(current_user());
+
+ for (i = 0; i < nr_args; i++) {
+ struct io_mapped_ubuf *imu = &ctx->user_bufs[i];
+ unsigned long off, start, end, ubuf;
+ int pret, nr_pages;
+ struct iovec iov;
+ size_t size;
+
+ ret = io_copy_iov(ctx, &iov, arg, i);
+ if (ret)
+ break;
+
+ /*
+ * Don't impose further limits on the size and buffer
+ * constraints here, we'll -EINVAL later when IO is
+ * submitted if they are wrong.
+ */
+ ret = -EFAULT;
+ if (!iov.iov_base || !iov.iov_len)
+ goto err;
+
+ /* arbitrary limit, but we need something */
+ if (iov.iov_len > SZ_1G)
+ goto err;
+
+ ubuf = (unsigned long) iov.iov_base;
+ end = (ubuf + iov.iov_len + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ start = ubuf >> PAGE_SHIFT;
+ nr_pages = end - start;
+
+ ret = io_account_mem(ctx, nr_pages);
+ if (ret)
+ goto err;
+
+ if (!pages || nr_pages > got_pages) {
+ kfree(pages);
+ pages = kmalloc_array(nr_pages, sizeof(struct page *),
+ GFP_KERNEL);
+ if (!pages) {
+ io_unaccount_mem(ctx, nr_pages);
+ goto err;
+ }
+ got_pages = nr_pages;
+ }
+
+ imu->bvec = kmalloc_array(nr_pages, sizeof(struct bio_vec),
+ GFP_KERNEL);
+ if (!imu->bvec) {
+ io_unaccount_mem(ctx, nr_pages);
+ goto err;
+ }
+
+ down_read(¤t->mm->mmap_sem);
+ pret = get_user_pages_longterm(ubuf, nr_pages,
+ FOLL_WRITE | FOLL_ANON, pages,
+ NULL);
+ up_read(¤t->mm->mmap_sem);
+
+ if (pret != nr_pages) {
+ if (pret > 0) {
+ for (j = 0; j < pret; j++)
+ put_page(pages[j]);
+ }
+ ret = pret < 0 ? pret : -EFAULT;
+ io_unaccount_mem(ctx, nr_pages);
+ goto err;
+ }
+
+ off = ubuf & ~PAGE_MASK;
+ size = iov.iov_len;
+ for (j = 0; j < nr_pages; j++) {
+ size_t vec_len;
+
+ vec_len = min_t(size_t, size, PAGE_SIZE - off);
+ imu->bvec[j].bv_page = pages[j];
+ imu->bvec[j].bv_len = vec_len;
+ imu->bvec[j].bv_offset = off;
+ off = 0;
+ size -= vec_len;
+ }
+ /* store original address for later verification */
+ imu->ubuf = ubuf;
+ imu->len = iov.iov_len;
+ imu->nr_bvecs = nr_pages;
+ }
+ kfree(pages);
+ ctx->nr_user_bufs = nr_args;
+ return 0;
+err:
+ kfree(pages);
+ io_sqe_buffer_unregister(ctx);
+ return ret;
+}
+
static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
destroy_workqueue(ctx->sqo_wq);
@@ -1225,6 +1487,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)
put_files_struct(ctx->sqo_files);
io_iopoll_reap_events(ctx);
+ io_sqe_buffer_unregister(ctx);
io_mem_free(ctx->sq_ring);
io_mem_free(ctx->sq_sqes);
@@ -1530,6 +1793,60 @@ SYSCALL_DEFINE2(io_uring_setup, u32, entries,
return io_uring_setup(entries, params);
}
+static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
+ void __user *arg, unsigned nr_args)
+{
+ int ret;
+
+ percpu_ref_kill(&ctx->refs);
+ wait_for_completion(&ctx->ctx_done);
+
+ switch (opcode) {
+ case IORING_REGISTER_BUFFERS:
+ ret = io_sqe_buffer_register(ctx, arg, nr_args);
+ break;
+ case IORING_UNREGISTER_BUFFERS:
+ ret = -EINVAL;
+ if (arg || nr_args)
+ break;
+ ret = io_sqe_buffer_unregister(ctx);
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ /* bring the ctx back to life */
+ reinit_completion(&ctx->ctx_done);
+ percpu_ref_reinit(&ctx->refs);
+ return ret;
+}
+
+SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
+ void __user *, arg, unsigned int, nr_args)
+{
+ struct io_ring_ctx *ctx;
+ long ret = -EBADF;
+ struct fd f;
+
+ f = fdget(fd);
+ if (!f.file)
+ return -EBADF;
+
+ ret = -EOPNOTSUPP;
+ if (f.file->f_op != &io_uring_fops)
+ goto out_fput;
+
+ ctx = f.file->private_data;
+
+ mutex_lock(&ctx->uring_lock);
+ ret = __io_uring_register(ctx, opcode, arg, nr_args);
+ mutex_unlock(&ctx->uring_lock);
+out_fput:
+ fdput(f);
+ return ret;
+}
+
static int __init io_uring_init(void)
{
req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
diff --git a/include/linux/sched/user.h b/include/linux/sched/user.h
index 39ad98c09c58..c7b5f86b91a1 100644
--- a/include/linux/sched/user.h
+++ b/include/linux/sched/user.h
@@ -40,7 +40,7 @@ struct user_struct {
kuid_t uid;
#if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) || \
- defined(CONFIG_NET)
+ defined(CONFIG_NET) || defined(CONFIG_IO_URING)
atomic_long_t locked_vm;
#endif
diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h
index 3072dbaa7869..3681c05ac538 100644
--- a/include/linux/syscalls.h
+++ b/include/linux/syscalls.h
@@ -315,6 +315,8 @@ asmlinkage long sys_io_uring_setup(u32 entries,
asmlinkage long sys_io_uring_enter(unsigned int fd, u32 to_submit,
u32 min_complete, u32 flags,
const sigset_t __user *sig, size_t sigsz);
+asmlinkage long sys_io_uring_register(unsigned int fd, unsigned int op,
+ void __user *arg, unsigned int nr_args);
/* fs/xattr.c */
asmlinkage long sys_setxattr(const char __user *path, const char __user *name,
diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h
index 87871e7b7ea7..d346229a1eb0 100644
--- a/include/uapi/asm-generic/unistd.h
+++ b/include/uapi/asm-generic/unistd.h
@@ -744,9 +744,11 @@ __SYSCALL(__NR_kexec_file_load, sys_kexec_file_load)
__SYSCALL(__NR_io_uring_setup, sys_io_uring_setup)
#define __NR_io_uring_enter 426
__SYSCALL(__NR_io_uring_enter, sys_io_uring_enter)
+#define __NR_io_uring_register 427
+__SYSCALL(__NR_io_uring_register, sys_io_uring_register)
#undef __NR_syscalls
-#define __NR_syscalls 427
+#define __NR_syscalls 428
/*
* 32 bit systems traditionally used different
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 4952fc921866..16c423d74f2e 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -27,7 +27,10 @@ struct io_uring_sqe {
__u32 fsync_flags;
};
__u64 user_data; /* data to be passed back at completion time */
- __u64 __pad2[3];
+ union {
+ __u16 buf_index; /* index into fixed buffers, if used */
+ __u64 __pad2[3];
+ };
};
/*
@@ -39,6 +42,8 @@ struct io_uring_sqe {
#define IORING_OP_READV 1
#define IORING_OP_WRITEV 2
#define IORING_OP_FSYNC 3
+#define IORING_OP_READ_FIXED 4
+#define IORING_OP_WRITE_FIXED 5
/*
* sqe->fsync_flags
@@ -102,4 +107,10 @@ struct io_uring_params {
struct io_cqring_offsets cq_off;
};
+/*
+ * io_uring_register(2) opcodes and arguments
+ */
+#define IORING_REGISTER_BUFFERS 0
+#define IORING_UNREGISTER_BUFFERS 1
+
#endif
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c
index ee5e523564bb..1bb6604dc19f 100644
--- a/kernel/sys_ni.c
+++ b/kernel/sys_ni.c
@@ -48,6 +48,7 @@ COND_SYSCALL_COMPAT(io_getevents);
COND_SYSCALL_COMPAT(io_pgetevents);
COND_SYSCALL(io_uring_setup);
COND_SYSCALL(io_uring_enter);
+COND_SYSCALL(io_uring_register);
/* fs/xattr.c */
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* [PATCH 13/18] io_uring: add file set registration
From: Jens Axboe @ 2019-02-01 15:24 UTC (permalink / raw)
To: linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh, Jens Axboe
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
We normally have to fget/fput for each IO we do on a file. Even with
the batching we do, the cost of the atomic inc/dec of the file usage
count adds up.
This adds IORING_REGISTER_FILES, and IORING_UNREGISTER_FILES opcodes
for the io_uring_register(2) system call. The arguments passed in must
be an array of __s32 holding file descriptors, and nr_args should hold
the number of file descriptors the application wishes to pin for the
duration of the io_uring context (or until IORING_UNREGISTER_FILES is
called).
When used, the application must set IOSQE_FIXED_FILE in the sqe->flags
member. Then, instead of setting sqe->fd to the real fd, it sets sqe->fd
to the index in the array passed in to IORING_REGISTER_FILES.
Files are automatically unregistered when the io_uring context is
torn down. An application need only unregister if it wishes to
register a new set of fds.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 147 +++++++++++++++++++++++++++++-----
include/uapi/linux/io_uring.h | 9 ++-
2 files changed, 133 insertions(+), 23 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 80c788d8c22a..61b58a046429 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -100,6 +100,14 @@ struct io_ring_ctx {
struct fasync_struct *cq_fasync;
} ____cacheline_aligned_in_smp;
+ /*
+ * If used, fixed file set. Writers must ensure that ->refs is dead,
+ * readers must ensure that ->refs is alive as long as the file* is
+ * used. Only updated through io_uring_register(2).
+ */
+ struct file **user_files;
+ unsigned nr_user_files;
+
/* if used, fixed mapped user buffers */
unsigned nr_user_bufs;
struct io_mapped_ubuf *user_bufs;
@@ -143,6 +151,7 @@ struct io_kiocb {
unsigned int flags;
#define REQ_F_FORCE_NONBLOCK 1 /* inline submission attempt */
#define REQ_F_IOPOLL_COMPLETED 2 /* polled IO has completed */
+#define REQ_F_FIXED_FILE 4 /* ctx owns file */
u64 user_data;
u64 error;
@@ -357,15 +366,17 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
* Batched puts of the same file, to avoid dirtying the
* file usage count multiple times, if avoidable.
*/
- if (!file) {
- file = req->rw.ki_filp;
- file_count = 1;
- } else if (file == req->rw.ki_filp) {
- file_count++;
- } else {
- fput_many(file, file_count);
- file = req->rw.ki_filp;
- file_count = 1;
+ if (!(req->flags & REQ_F_FIXED_FILE)) {
+ if (!file) {
+ file = req->rw.ki_filp;
+ file_count = 1;
+ } else if (file == req->rw.ki_filp) {
+ file_count++;
+ } else {
+ fput_many(file, file_count);
+ file = req->rw.ki_filp;
+ file_count = 1;
+ }
}
if (to_free == ARRAY_SIZE(reqs))
@@ -497,13 +508,19 @@ static void kiocb_end_write(struct kiocb *kiocb)
}
}
+static void io_fput(struct io_kiocb *req)
+{
+ if (!(req->flags & REQ_F_FIXED_FILE))
+ fput(req->rw.ki_filp);
+}
+
static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
kiocb_end_write(kiocb);
- fput(kiocb->ki_filp);
+ io_fput(req);
io_cqring_add_event(req->ctx, req->user_data, res, 0);
io_free_req(req);
}
@@ -602,11 +619,22 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
{
struct io_ring_ctx *ctx = req->ctx;
struct kiocb *kiocb = &req->rw;
- unsigned ioprio;
+ unsigned ioprio, flags;
int fd, ret;
+ flags = READ_ONCE(sqe->flags);
fd = READ_ONCE(sqe->fd);
- kiocb->ki_filp = io_file_get(state, fd);
+
+ if (flags & IOSQE_FIXED_FILE) {
+ if (unlikely(!ctx->user_files ||
+ (unsigned) fd >= ctx->nr_user_files))
+ return -EBADF;
+ kiocb->ki_filp = ctx->user_files[fd];
+ req->flags |= REQ_F_FIXED_FILE;
+ } else {
+ kiocb->ki_filp = io_file_get(state, fd);
+ }
+
if (unlikely(!kiocb->ki_filp))
return -EBADF;
kiocb->ki_pos = READ_ONCE(sqe->off);
@@ -648,10 +676,14 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
}
return 0;
out_fput:
- /* in case of error, we didn't use this file reference. drop it. */
- if (state)
- state->used_refs--;
- io_file_put(state, kiocb->ki_filp);
+ if (!(flags & IOSQE_FIXED_FILE)) {
+ /*
+ * in case of error, we didn't use this file reference. drop it.
+ */
+ if (state)
+ state->used_refs--;
+ io_file_put(state, kiocb->ki_filp);
+ }
return ret;
}
@@ -782,7 +814,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct sqe_submit *s,
kfree(iovec);
out_fput:
if (unlikely(ret))
- fput(file);
+ io_fput(req);
return ret;
}
@@ -837,7 +869,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct sqe_submit *s,
kfree(iovec);
out_fput:
if (unlikely(ret))
- fput(file);
+ io_fput(req);
return ret;
}
@@ -863,7 +895,7 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
loff_t sqe_off = READ_ONCE(sqe->off);
loff_t sqe_len = READ_ONCE(sqe->len);
loff_t end = sqe_off + sqe_len;
- unsigned fsync_flags;
+ unsigned fsync_flags, flags;
struct file *file;
int ret, fd;
@@ -881,14 +913,23 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
return -EINVAL;
fd = READ_ONCE(sqe->fd);
- file = fget(fd);
+ flags = READ_ONCE(sqe->flags);
+
+ if (flags & IOSQE_FIXED_FILE) {
+ if (unlikely(!ctx->user_files || fd >= ctx->nr_user_files))
+ return -EBADF;
+ file = ctx->user_files[fd];
+ } else {
+ file = fget(fd);
+ }
if (unlikely(!file))
return -EBADF;
ret = vfs_fsync_range(file, sqe_off, end > 0 ? end : LLONG_MAX,
fsync_flags & IORING_FSYNC_DATASYNC);
- fput(file);
+ if (!(flags & IOSQE_FIXED_FILE))
+ fput(file);
io_cqring_add_event(ctx, sqe->user_data, ret, 0);
io_free_req(req);
return 0;
@@ -1034,7 +1075,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, const struct sqe_submit *s,
ssize_t ret;
/* enforce forwards compatibility on users */
- if (unlikely(s->sqe->flags))
+ if (unlikely(s->sqe->flags & ~IOSQE_FIXED_FILE))
return -EINVAL;
req = io_get_req(ctx, state);
@@ -1220,6 +1261,58 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
return ring->r.head == ring->r.tail ? ret : 0;
}
+static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
+{
+ int i;
+
+ if (!ctx->user_files)
+ return -ENXIO;
+
+ for (i = 0; i < ctx->nr_user_files; i++)
+ fput(ctx->user_files[i]);
+
+ kfree(ctx->user_files);
+ ctx->user_files = NULL;
+ ctx->nr_user_files = 0;
+ return 0;
+}
+
+static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
+ unsigned nr_args)
+{
+ __s32 __user *fds = (__s32 __user *) arg;
+ int fd, ret = 0;
+ unsigned i;
+
+ if (ctx->user_files)
+ return -EBUSY;
+ if (!nr_args)
+ return -EINVAL;
+
+ ctx->user_files = kcalloc(nr_args, sizeof(struct file *), GFP_KERNEL);
+ if (!ctx->user_files)
+ return -ENOMEM;
+
+ for (i = 0; i < nr_args; i++) {
+ ret = -EFAULT;
+ if (copy_from_user(&fd, &fds[i], sizeof(fd)))
+ break;
+
+ ctx->user_files[i] = fget(fd);
+
+ ret = -EBADF;
+ if (!ctx->user_files[i])
+ break;
+ ctx->nr_user_files++;
+ ret = 0;
+ }
+
+ if (ret)
+ io_sqe_files_unregister(ctx);
+
+ return ret;
+}
+
static int io_sq_offload_start(struct io_ring_ctx *ctx)
{
int ret;
@@ -1488,6 +1581,7 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx)
io_iopoll_reap_events(ctx);
io_sqe_buffer_unregister(ctx);
+ io_sqe_files_unregister(ctx);
io_mem_free(ctx->sq_ring);
io_mem_free(ctx->sq_sqes);
@@ -1811,6 +1905,15 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
break;
ret = io_sqe_buffer_unregister(ctx);
break;
+ case IORING_REGISTER_FILES:
+ ret = io_sqe_files_register(ctx, arg, nr_args);
+ break;
+ case IORING_UNREGISTER_FILES:
+ ret = -EINVAL;
+ if (arg || nr_args)
+ break;
+ ret = io_sqe_files_unregister(ctx);
+ break;
default:
ret = -EINVAL;
break;
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 16c423d74f2e..3e79feb34a9c 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -16,7 +16,7 @@
*/
struct io_uring_sqe {
__u8 opcode; /* type of operation for this sqe */
- __u8 flags; /* as of now unused */
+ __u8 flags; /* IOSQE_ flags */
__u16 ioprio; /* ioprio for the request */
__s32 fd; /* file descriptor to do IO on */
__u64 off; /* offset into file */
@@ -33,6 +33,11 @@ struct io_uring_sqe {
};
};
+/*
+ * sqe->flags
+ */
+#define IOSQE_FIXED_FILE (1U << 0) /* use fixed fileset */
+
/*
* io_uring_setup() flags
*/
@@ -112,5 +117,7 @@ struct io_uring_params {
*/
#define IORING_REGISTER_BUFFERS 0
#define IORING_UNREGISTER_BUFFERS 1
+#define IORING_REGISTER_FILES 2
+#define IORING_UNREGISTER_FILES 3
#endif
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* [PATCH 14/18] io_uring: add submission polling
From: Jens Axboe @ 2019-02-01 15:24 UTC (permalink / raw)
To: linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh, Jens Axboe
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
This enables an application to do IO, without ever entering the kernel.
By using the SQ ring to fill in new sqes and watching for completions
on the CQ ring, we can submit and reap IOs without doing a single system
call. The kernel side thread will poll for new submissions, and in case
of HIPRI/polled IO, it'll also poll for completions.
By default, we allow 1 second of active spinning. This can by changed
by passing in a different grace period at io_uring_register(2) time.
If the thread exceeds this idle time without having any work to do, it
will set:
sq_ring->flags |= IORING_SQ_NEED_WAKEUP.
The application will have to call io_uring_enter() to start things back
up again. If IO is kept busy, that will never be needed. Basically an
application that has this feature enabled will guard it's
io_uring_enter(2) call with:
read_barrier();
if (*sq_ring->flags & IORING_SQ_NEED_WAKEUP)
io_uring_enter(fd, 0, 0, IORING_ENTER_SQ_WAKEUP);
instead of calling it unconditionally.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 247 +++++++++++++++++++++++++++++++++-
include/uapi/linux/io_uring.h | 12 +-
2 files changed, 253 insertions(+), 6 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 61b58a046429..8f8cde20bb6c 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -24,6 +24,7 @@
#include <linux/percpu.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
+#include <linux/kthread.h>
#include <linux/blkdev.h>
#include <linux/bvec.h>
#include <linux/anon_inodes.h>
@@ -81,14 +82,17 @@ struct io_ring_ctx {
unsigned cached_sq_head;
unsigned sq_entries;
unsigned sq_mask;
- unsigned sq_thread_cpu;
+ unsigned sq_thread_idle;
struct io_uring_sqe *sq_sqes;
} ____cacheline_aligned_in_smp;
/* IO offload */
struct workqueue_struct *sqo_wq;
+ struct task_struct *sqo_thread; /* if using sq thread polling */
struct mm_struct *sqo_mm;
struct files_struct *sqo_files;
+ wait_queue_head_t sqo_wait;
+ unsigned sqo_stop;
struct {
/* CQ ring */
@@ -278,6 +282,8 @@ static void io_cqring_add_event(struct io_ring_ctx *ctx, u64 ki_user_data,
if (waitqueue_active(&ctx->wait))
wake_up(&ctx->wait);
+ if (waitqueue_active(&ctx->sqo_wait))
+ wake_up(&ctx->sqo_wait);
}
static void io_ring_drop_ctx_refs(struct io_ring_ctx *ctx, unsigned refs)
@@ -1179,6 +1185,179 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
return false;
}
+static int io_submit_sqes(struct io_ring_ctx *ctx, struct sqe_submit *sqes,
+ unsigned int nr, bool has_user, bool mm_fault)
+{
+ struct io_submit_state state, *statep = NULL;
+ int ret, i, submitted = 0;
+
+ if (nr > IO_PLUG_THRESHOLD) {
+ io_submit_state_start(&state, ctx, nr);
+ statep = &state;
+ }
+
+ for (i = 0; i < nr; i++) {
+ if (unlikely(mm_fault)) {
+ ret = -EFAULT;
+ } else {
+ sqes[i].has_user = has_user;
+ sqes[i].needs_lock = true;
+ ret = io_submit_sqe(ctx, &sqes[i], statep);
+ }
+ if (!ret) {
+ submitted++;
+ continue;
+ }
+
+ io_cqring_add_event(ctx, sqes[i].sqe->user_data, ret, 0);
+ }
+
+ if (statep)
+ io_submit_state_end(&state);
+
+ return submitted;
+}
+
+static int io_sq_thread(void *data)
+{
+ struct sqe_submit sqes[IO_IOPOLL_BATCH];
+ struct io_ring_ctx *ctx = data;
+ struct mm_struct *cur_mm = NULL;
+ struct files_struct *old_files;
+ mm_segment_t old_fs;
+ DEFINE_WAIT(wait);
+ unsigned inflight;
+ unsigned long timeout;
+
+ task_lock(current);
+ old_files = current->files;
+ current->files = ctx->sqo_files;
+ task_unlock(current);
+
+ old_fs = get_fs();
+ set_fs(USER_DS);
+
+ timeout = inflight = 0;
+ while (!kthread_should_stop() && !ctx->sqo_stop) {
+ bool all_fixed, mm_fault = false;
+ int i;
+
+ if (inflight) {
+ unsigned nr_events = 0;
+
+ if (ctx->flags & IORING_SETUP_IOPOLL) {
+ /*
+ * We disallow the app entering submit/complete
+ * with polling, but we still need to lock the
+ * ring to prevent racing with polled issue
+ * that got punted to a workqueue.
+ */
+ mutex_lock(&ctx->uring_lock);
+ io_iopoll_check(ctx, &nr_events, 0);
+ mutex_unlock(&ctx->uring_lock);
+ } else {
+ /*
+ * Normal IO, just pretend everything completed.
+ * We don't have to poll completions for that.
+ */
+ nr_events = inflight;
+ }
+
+ inflight -= nr_events;
+ if (!inflight)
+ timeout = jiffies + ctx->sq_thread_idle;
+ }
+
+ if (!io_get_sqring(ctx, &sqes[0])) {
+ /*
+ * We're polling. If we're within the defined idle
+ * period, then let us spin without work before going
+ * to sleep.
+ */
+ if (inflight || !time_after(jiffies, timeout)) {
+ cpu_relax();
+ continue;
+ }
+
+ /*
+ * Drop cur_mm before scheduling, we can't hold it for
+ * long periods (or over schedule()). Do this before
+ * adding ourselves to the waitqueue, as the unuse/drop
+ * may sleep.
+ */
+ if (cur_mm) {
+ unuse_mm(cur_mm);
+ mmput(cur_mm);
+ cur_mm = NULL;
+ }
+
+ prepare_to_wait(&ctx->sqo_wait, &wait,
+ TASK_INTERRUPTIBLE);
+
+ /* Tell userspace we may need a wakeup call */
+ ctx->sq_ring->flags |= IORING_SQ_NEED_WAKEUP;
+ smp_wmb();
+
+ if (!io_get_sqring(ctx, &sqes[0])) {
+ if (kthread_should_stop()) {
+ finish_wait(&ctx->sqo_wait, &wait);
+ break;
+ }
+ if (signal_pending(current))
+ flush_signals(current);
+ schedule();
+ finish_wait(&ctx->sqo_wait, &wait);
+
+ ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP;
+ smp_wmb();
+ continue;
+ }
+ finish_wait(&ctx->sqo_wait, &wait);
+
+ ctx->sq_ring->flags &= ~IORING_SQ_NEED_WAKEUP;
+ smp_wmb();
+ }
+
+ i = 0;
+ all_fixed = true;
+ do {
+ if (all_fixed && io_sqe_needs_user(sqes[i].sqe))
+ all_fixed = false;
+
+ i++;
+ if (i == ARRAY_SIZE(sqes))
+ break;
+ } while (io_get_sqring(ctx, &sqes[i]));
+
+ io_commit_sqring(ctx);
+
+ /* Unless all new commands are FIXED regions, grab mm */
+ if (!all_fixed && !cur_mm) {
+ mm_fault = !mmget_not_zero(ctx->sqo_mm);
+ if (!mm_fault) {
+ use_mm(ctx->sqo_mm);
+ cur_mm = ctx->sqo_mm;
+ }
+ }
+
+ inflight += io_submit_sqes(ctx, sqes, i, cur_mm != NULL,
+ mm_fault);
+ }
+
+ io_iopoll_reap_events(ctx);
+
+ task_lock(current);
+ current->files = old_files;
+ task_unlock(current);
+
+ set_fs(old_fs);
+ if (cur_mm) {
+ unuse_mm(cur_mm);
+ mmput(cur_mm);
+ }
+ return 0;
+}
+
static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit)
{
struct io_submit_state state, *statep = NULL;
@@ -1313,10 +1492,12 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
return ret;
}
-static int io_sq_offload_start(struct io_ring_ctx *ctx)
+static int io_sq_offload_start(struct io_ring_ctx *ctx,
+ struct io_uring_params *p)
{
int ret;
+ init_waitqueue_head(&ctx->sqo_wait);
mmgrab(current->mm);
ctx->sqo_mm = current->mm;
@@ -1325,6 +1506,38 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)
if (!ctx->sqo_files)
goto err;
+ ctx->sq_thread_idle = msecs_to_jiffies(p->sq_thread_idle);
+ if (!ctx->sq_thread_idle)
+ ctx->sq_thread_idle = HZ;
+
+ ret = -EINVAL;
+ if (!cpu_possible(p->sq_thread_cpu))
+ goto err;
+
+ if (ctx->flags & IORING_SETUP_SQPOLL) {
+ if (p->flags & IORING_SETUP_SQ_AFF) {
+ int cpu;
+
+ cpu = array_index_nospec(p->sq_thread_cpu, NR_CPUS);
+ ctx->sqo_thread = kthread_create_on_cpu(io_sq_thread,
+ ctx, cpu,
+ "io_uring-sq");
+ } else {
+ ctx->sqo_thread = kthread_create(io_sq_thread, ctx,
+ "io_uring-sq");
+ }
+ if (IS_ERR(ctx->sqo_thread)) {
+ ret = PTR_ERR(ctx->sqo_thread);
+ ctx->sqo_thread = NULL;
+ goto err;
+ }
+ wake_up_process(ctx->sqo_thread);
+ } else if (p->flags & IORING_SETUP_SQ_AFF) {
+ /* Can't have SQ_AFF without SQPOLL */
+ ret = -EINVAL;
+ goto err;
+ }
+
/* Do QD, or 2 * CPUS, whatever is smallest */
ctx->sqo_wq = alloc_workqueue("io_ring-wq", WQ_UNBOUND | WQ_FREEZABLE,
min(ctx->sq_entries - 1, 2 * num_online_cpus()));
@@ -1335,6 +1548,12 @@ static int io_sq_offload_start(struct io_ring_ctx *ctx)
return 0;
err:
+ if (ctx->sqo_thread) {
+ ctx->sqo_stop = 1;
+ mb();
+ kthread_stop(ctx->sqo_thread);
+ ctx->sqo_thread = NULL;
+ }
if (ctx->sqo_files) {
put_files_struct(ctx->sqo_files);
ctx->sqo_files = NULL;
@@ -1575,6 +1794,11 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg,
static void io_ring_ctx_free(struct io_ring_ctx *ctx)
{
+ if (ctx->sqo_thread) {
+ ctx->sqo_stop = 1;
+ mb();
+ kthread_stop(ctx->sqo_thread);
+ }
destroy_workqueue(ctx->sqo_wq);
mmdrop(ctx->sqo_mm);
put_files_struct(ctx->sqo_files);
@@ -1674,7 +1898,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
int submitted = 0;
struct fd f;
- if (flags & ~IORING_ENTER_GETEVENTS)
+ if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP))
return -EINVAL;
f = fdget(fd);
@@ -1690,6 +1914,18 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
if (!percpu_ref_tryget(&ctx->refs))
goto out_fput;
+ /*
+ * For SQ polling, the thread will do all submissions and completions.
+ * Just return the requested submit count, and wake the thread if
+ * we were asked to.
+ */
+ if (ctx->flags & IORING_SETUP_SQPOLL) {
+ if (flags & IORING_ENTER_SQ_WAKEUP)
+ wake_up(&ctx->sqo_wait);
+ submitted = to_submit;
+ goto out_ctx;
+ }
+
if (to_submit) {
to_submit = min(to_submit, ctx->sq_entries);
@@ -1819,7 +2055,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p)
if (ret)
goto err;
- ret = io_sq_offload_start(ctx);
+ ret = io_sq_offload_start(ctx, p);
if (ret)
goto err;
@@ -1868,7 +2104,8 @@ static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
return -EINVAL;
}
- if (p.flags & ~IORING_SETUP_IOPOLL)
+ if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
+ IORING_SETUP_SQ_AFF))
return -EINVAL;
ret = io_uring_create(entries, &p);
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 3e79feb34a9c..0d85da31e260 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -42,6 +42,8 @@ struct io_uring_sqe {
* io_uring_setup() flags
*/
#define IORING_SETUP_IOPOLL (1U << 0) /* io_context is polled */
+#define IORING_SETUP_SQPOLL (1U << 1) /* SQ poll thread */
+#define IORING_SETUP_SQ_AFF (1U << 2) /* sq_thread_cpu is valid */
#define IORING_OP_NOP 0
#define IORING_OP_READV 1
@@ -85,6 +87,11 @@ struct io_sqring_offsets {
__u32 resv[3];
};
+/*
+ * sq_ring->flags
+ */
+#define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */
+
struct io_cqring_offsets {
__u32 head;
__u32 tail;
@@ -99,6 +106,7 @@ struct io_cqring_offsets {
* io_uring_enter(2) flags
*/
#define IORING_ENTER_GETEVENTS (1U << 0)
+#define IORING_ENTER_SQ_WAKEUP (1U << 1)
/*
* Passed in for io_uring_setup(2). Copied back with updated info on success
@@ -107,7 +115,9 @@ struct io_uring_params {
__u32 sq_entries;
__u32 cq_entries;
__u32 flags;
- __u32 resv[7];
+ __u32 sq_thread_cpu;
+ __u32 sq_thread_idle;
+ __u32 resv[5];
struct io_sqring_offsets sq_off;
struct io_cqring_offsets cq_off;
};
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* [PATCH 15/18] io_uring: add io_kiocb ref count
From: Jens Axboe @ 2019-02-01 15:24 UTC (permalink / raw)
To: linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh, Jens Axboe
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
We'll use this for the POLL implementation. Regular requests will
NOT be using references, so initialize it to 0. Any real use of
the io_kiocb ref will initialize it to at least 2.
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 8f8cde20bb6c..97ec812b10b3 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -153,6 +153,7 @@ struct io_kiocb {
struct io_ring_ctx *ctx;
struct list_head list;
unsigned int flags;
+ refcount_t refs;
#define REQ_F_FORCE_NONBLOCK 1 /* inline submission attempt */
#define REQ_F_IOPOLL_COMPLETED 2 /* polled IO has completed */
#define REQ_F_FIXED_FILE 4 /* ctx owns file */
@@ -326,6 +327,7 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
req->ctx = ctx;
req->flags = 0;
+ refcount_set(&req->refs, 0);
return req;
out:
io_ring_drop_ctx_refs(ctx, 1);
@@ -343,8 +345,10 @@ static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)
static void io_free_req(struct io_kiocb *req)
{
- io_ring_drop_ctx_refs(req->ctx, 1);
- kmem_cache_free(req_cachep, req);
+ if (!refcount_read(&req->refs) || refcount_dec_and_test(&req->refs)) {
+ io_ring_drop_ctx_refs(req->ctx, 1);
+ kmem_cache_free(req_cachep, req);
+ }
}
/*
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* [PATCH 16/18] io_uring: add support for IORING_OP_POLL
From: Jens Axboe @ 2019-02-01 15:24 UTC (permalink / raw)
To: linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh, Jens Axboe
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
This is basically a direct port of bfe4037e722e, which implements a
one-shot poll command through aio. Description below is based on that
commit as well. However, instead of adding a POLL command and relying
on io_cancel(2) to remove it, we mimic the epoll(2) interface of
having a command to add a poll notification, IORING_OP_POLL_ADD,
and one to remove it again, IORING_OP_POLL_REMOVE.
To poll for a file descriptor the application should submit an sqe of
type IORING_OP_POLL. It will poll the fd for the events specified in the
poll_events field.
Unlike poll or epoll without EPOLLONESHOT this interface always works in
one shot mode, that is once the sqe is completed, it will have to be
resubmitted.
Based-on-code-from: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 256 +++++++++++++++++++++++++++++++++-
include/uapi/linux/io_uring.h | 3 +
2 files changed, 258 insertions(+), 1 deletion(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 97ec812b10b3..fdfcd0c75278 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -135,6 +135,7 @@ struct io_ring_ctx {
* manipulate the list, hence no extra locking is needed there.
*/
struct list_head poll_list;
+ struct list_head cancel_list;
} ____cacheline_aligned_in_smp;
};
@@ -145,8 +146,20 @@ struct sqe_submit {
bool needs_lock;
};
+struct io_poll_iocb {
+ struct file *file;
+ struct wait_queue_head *head;
+ __poll_t events;
+ bool woken;
+ bool canceled;
+ struct wait_queue_entry wait;
+};
+
struct io_kiocb {
- struct kiocb rw;
+ union {
+ struct kiocb rw;
+ struct io_poll_iocb poll;
+ };
struct sqe_submit submit;
@@ -217,6 +230,7 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
init_waitqueue_head(&ctx->wait);
spin_lock_init(&ctx->completion_lock);
INIT_LIST_HEAD(&ctx->poll_list);
+ INIT_LIST_HEAD(&ctx->cancel_list);
return ctx;
}
@@ -945,6 +959,239 @@ static int io_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe,
return 0;
}
+static void io_poll_remove_one(struct io_kiocb *req)
+{
+ struct io_poll_iocb *poll = &req->poll;
+
+ spin_lock(&poll->head->lock);
+ WRITE_ONCE(poll->canceled, true);
+ if (!list_empty(&poll->wait.entry)) {
+ list_del_init(&poll->wait.entry);
+ queue_work(req->ctx->sqo_wq, &req->work);
+ }
+ spin_unlock(&poll->head->lock);
+
+ list_del_init(&req->list);
+}
+
+static void io_poll_remove_all(struct io_ring_ctx *ctx)
+{
+ struct io_kiocb *req;
+
+ spin_lock_irq(&ctx->completion_lock);
+ while (!list_empty(&ctx->cancel_list)) {
+ req = list_first_entry(&ctx->cancel_list, struct io_kiocb,list);
+ io_poll_remove_one(req);
+ }
+ spin_unlock_irq(&ctx->completion_lock);
+}
+
+/*
+ * Find a running poll command that matches one specified in sqe->addr,
+ * and remove it if found.
+ */
+static int io_poll_remove(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+{
+ struct io_ring_ctx *ctx = req->ctx;
+ struct io_kiocb *poll_req, *next;
+ int ret = -ENOENT;
+
+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+ return -EINVAL;
+ if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
+ sqe->poll_events)
+ return -EINVAL;
+
+ spin_lock_irq(&ctx->completion_lock);
+ list_for_each_entry_safe(poll_req, next, &ctx->cancel_list, list) {
+ if (READ_ONCE(sqe->addr) == poll_req->user_data) {
+ io_poll_remove_one(poll_req);
+ ret = 0;
+ break;
+ }
+ }
+ spin_unlock_irq(&ctx->completion_lock);
+
+ io_cqring_add_event(req->ctx, sqe->user_data, ret, 0);
+ io_free_req(req);
+ return 0;
+}
+
+static void io_poll_complete(struct io_kiocb *req, __poll_t mask)
+{
+ io_cqring_add_event(req->ctx, req->user_data, mangle_poll(mask), 0);
+ io_fput(req);
+ io_free_req(req);
+}
+
+static void io_poll_complete_work(struct work_struct *work)
+{
+ struct io_kiocb *req = container_of(work, struct io_kiocb, work);
+ struct io_poll_iocb *poll = &req->poll;
+ struct poll_table_struct pt = { ._key = poll->events };
+ struct io_ring_ctx *ctx = req->ctx;
+ __poll_t mask = 0;
+
+ if (!READ_ONCE(poll->canceled))
+ mask = vfs_poll(poll->file, &pt) & poll->events;
+
+ /*
+ * Note that ->ki_cancel callers also delete iocb from active_reqs after
+ * calling ->ki_cancel. We need the ctx_lock roundtrip here to
+ * synchronize with them. In the cancellation case the list_del_init
+ * itself is not actually needed, but harmless so we keep it in to
+ * avoid further branches in the fast path.
+ */
+ spin_lock_irq(&ctx->completion_lock);
+ if (!mask && !READ_ONCE(poll->canceled)) {
+ add_wait_queue(poll->head, &poll->wait);
+ spin_unlock_irq(&ctx->completion_lock);
+ return;
+ }
+ list_del_init(&req->list);
+ spin_unlock_irq(&ctx->completion_lock);
+
+ io_poll_complete(req, mask);
+}
+
+static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
+ void *key)
+{
+ struct io_poll_iocb *poll = container_of(wait, struct io_poll_iocb,
+ wait);
+ struct io_kiocb *req = container_of(poll, struct io_kiocb, poll);
+ struct io_ring_ctx *ctx = req->ctx;
+ __poll_t mask = key_to_poll(key);
+
+ poll->woken = true;
+
+ /* for instances that support it check for an event match first: */
+ if (mask) {
+ if (!(mask & poll->events))
+ return 0;
+
+ /* try to complete the iocb inline if we can: */
+ if (spin_trylock(&ctx->completion_lock)) {
+ list_del(&req->list);
+ spin_unlock(&ctx->completion_lock);
+
+ list_del_init(&poll->wait.entry);
+ io_poll_complete(req, mask);
+ return 1;
+ }
+ }
+
+ list_del_init(&poll->wait.entry);
+ queue_work(ctx->sqo_wq, &req->work);
+ return 1;
+}
+
+struct io_poll_table {
+ struct poll_table_struct pt;
+ struct io_kiocb *req;
+ int error;
+};
+
+static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
+ struct poll_table_struct *p)
+{
+ struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
+
+ if (unlikely(pt->req->poll.head)) {
+ pt->error = -EINVAL;
+ return;
+ }
+
+ pt->error = 0;
+ pt->req->poll.head = head;
+ add_wait_queue(head, &pt->req->poll.wait);
+}
+
+static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+{
+ struct io_poll_iocb *poll = &req->poll;
+ struct io_ring_ctx *ctx = req->ctx;
+ struct io_poll_table ipt;
+ unsigned flags;
+ __poll_t mask;
+ u16 events;
+ int fd;
+
+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+ return -EINVAL;
+ if (sqe->addr || sqe->ioprio || sqe->off || sqe->len || sqe->buf_index)
+ return -EINVAL;
+
+ INIT_WORK(&req->work, io_poll_complete_work);
+ events = READ_ONCE(sqe->poll_events);
+ poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
+
+ flags = READ_ONCE(sqe->flags);
+ fd = READ_ONCE(sqe->fd);
+
+ if (flags & IOSQE_FIXED_FILE) {
+ if (unlikely(!ctx->user_files || fd >= ctx->nr_user_files))
+ return -EBADF;
+ poll->file = ctx->user_files[fd];
+ req->flags |= REQ_F_FIXED_FILE;
+ } else {
+ poll->file = fget(fd);
+ }
+ if (unlikely(!poll->file))
+ return -EBADF;
+
+ poll->head = NULL;
+ poll->woken = false;
+ poll->canceled = false;
+
+ ipt.pt._qproc = io_poll_queue_proc;
+ ipt.pt._key = poll->events;
+ ipt.req = req;
+ ipt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */
+
+ /* initialized the list so that we can do list_empty checks */
+ INIT_LIST_HEAD(&poll->wait.entry);
+ init_waitqueue_func_entry(&poll->wait, io_poll_wake);
+
+ /* one for removal from waitqueue, one for this function */
+ refcount_set(&req->refs, 2);
+
+ mask = vfs_poll(poll->file, &ipt.pt) & poll->events;
+ if (unlikely(!poll->head)) {
+ /* we did not manage to set up a waitqueue, done */
+ goto out;
+ }
+
+ spin_lock_irq(&ctx->completion_lock);
+ spin_lock(&poll->head->lock);
+ if (poll->woken) {
+ /* wake_up context handles the rest */
+ mask = 0;
+ ipt.error = 0;
+ } else if (mask || ipt.error) {
+ /* if we get an error or a mask we are done */
+ WARN_ON_ONCE(list_empty(&poll->wait.entry));
+ list_del_init(&poll->wait.entry);
+ } else {
+ /* actually waiting for an event */
+ list_add_tail(&req->list, &ctx->cancel_list);
+ }
+ spin_unlock(&poll->head->lock);
+ spin_unlock_irq(&ctx->completion_lock);
+
+out:
+ if (unlikely(ipt.error)) {
+ if (!(flags & IOSQE_FIXED_FILE))
+ fput(poll->file);
+ return ipt.error;
+ }
+
+ if (mask)
+ io_poll_complete(req, mask);
+ io_free_req(req);
+ return 0;
+}
+
static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
const struct sqe_submit *s, bool force_nonblock,
struct io_submit_state *state)
@@ -980,6 +1227,12 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
case IORING_OP_FSYNC:
ret = io_fsync(req, s->sqe, force_nonblock);
break;
+ case IORING_OP_POLL_ADD:
+ ret = io_poll_add(req, s->sqe);
+ break;
+ case IORING_OP_POLL_REMOVE:
+ ret = io_poll_remove(req, s->sqe);
+ break;
default:
ret = -EINVAL;
break;
@@ -1848,6 +2101,7 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
percpu_ref_kill(&ctx->refs);
mutex_unlock(&ctx->uring_lock);
+ io_poll_remove_all(ctx);
io_iopoll_reap_events(ctx);
wait_for_completion(&ctx->ctx_done);
io_ring_ctx_free(ctx);
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 0d85da31e260..d319b2cd6319 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -25,6 +25,7 @@ struct io_uring_sqe {
union {
__kernel_rwf_t rw_flags;
__u32 fsync_flags;
+ __u16 poll_events;
};
__u64 user_data; /* data to be passed back at completion time */
union {
@@ -51,6 +52,8 @@ struct io_uring_sqe {
#define IORING_OP_FSYNC 3
#define IORING_OP_READ_FIXED 4
#define IORING_OP_WRITE_FIXED 5
+#define IORING_OP_POLL_ADD 6
+#define IORING_OP_POLL_REMOVE 7
/*
* sqe->fsync_flags
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* [PATCH 17/18] io_uring: allow workqueue item to handle multiple buffered requests
From: Jens Axboe @ 2019-02-01 15:24 UTC (permalink / raw)
To: linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh, Jens Axboe
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
Right now we punt any buffered request that ends up triggering an
-EAGAIN to an async workqueue. This works fine in terms of providing
async execution of them, but it also can create quite a lot of work
queue items. For sequentially buffered IO, it's advantageous to
serialize the issue of them. For reads, the first one will trigger a
read-ahead, and subsequent request merely end up waiting on later pages
to complete. For writes, devices usually respond better to streamed
sequential writes.
Add state to track the last buffered request we punted to a work queue,
and if the next one is sequential to the previous, attempt to get the
previous work item to handle it. We limit the number of sequential
add-ons to the a multiple (8) of the max read-ahead size of the file.
This should be a good number for both reads and wries, as it defines the
max IO size the device can do directly.
This drastically cuts down on the number of context switches we need to
handle buffered sequential IO, and a basic test case of copying a big
file with io_uring sees a 5x speedup.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 265 ++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 214 insertions(+), 51 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index fdfcd0c75278..0e700e81ac43 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -68,6 +68,16 @@ struct io_mapped_ubuf {
unsigned int nr_bvecs;
};
+struct async_list {
+ spinlock_t lock;
+ atomic_t cnt;
+ struct list_head list;
+
+ struct file *file;
+ off_t io_end;
+ size_t io_pages;
+};
+
struct io_ring_ctx {
struct {
struct percpu_ref refs;
@@ -137,6 +147,8 @@ struct io_ring_ctx {
struct list_head poll_list;
struct list_head cancel_list;
} ____cacheline_aligned_in_smp;
+
+ struct async_list pending_async[2];
};
struct sqe_submit {
@@ -170,6 +182,7 @@ struct io_kiocb {
#define REQ_F_FORCE_NONBLOCK 1 /* inline submission attempt */
#define REQ_F_IOPOLL_COMPLETED 2 /* polled IO has completed */
#define REQ_F_FIXED_FILE 4 /* ctx owns file */
+#define REQ_F_SEQ_PREV 8 /* sequential with previous */
u64 user_data;
u64 error;
@@ -213,6 +226,7 @@ static void io_ring_ctx_ref_free(struct percpu_ref *ref)
static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
{
struct io_ring_ctx *ctx;
+ int i;
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx)
@@ -228,6 +242,11 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
init_completion(&ctx->ctx_done);
mutex_init(&ctx->uring_lock);
init_waitqueue_head(&ctx->wait);
+ for (i = 0; i < ARRAY_SIZE(ctx->pending_async); i++) {
+ spin_lock_init(&ctx->pending_async[i].lock);
+ INIT_LIST_HEAD(&ctx->pending_async[i].list);
+ atomic_set(&ctx->pending_async[i].cnt, 0);
+ }
spin_lock_init(&ctx->completion_lock);
INIT_LIST_HEAD(&ctx->poll_list);
INIT_LIST_HEAD(&ctx->cancel_list);
@@ -799,6 +818,39 @@ static int io_import_iovec(struct io_ring_ctx *ctx, int rw,
return import_iovec(rw, buf, sqe_len, UIO_FASTIOV, iovec, iter);
}
+static void io_async_list_note(int rw, struct io_kiocb *req, size_t len)
+{
+ struct async_list *async_list = &req->ctx->pending_async[rw];
+ struct kiocb *kiocb = &req->rw;
+ struct file *filp = kiocb->ki_filp;
+ off_t io_end = kiocb->ki_pos + len;
+
+ if (filp == async_list->file && kiocb->ki_pos == async_list->io_end) {
+ unsigned long max_pages;
+
+ /* Use 8x RA size as a decent limiter for both reads/writes */
+ max_pages = filp->f_ra.ra_pages;
+ if (!max_pages)
+ max_pages = VM_MAX_READAHEAD >> (PAGE_SHIFT - 10);
+ max_pages *= 8;
+
+ len >>= PAGE_SHIFT;
+ if (async_list->io_pages + len <= max_pages) {
+ req->flags |= REQ_F_SEQ_PREV;
+ async_list->io_pages += len;
+ } else {
+ io_end = 0;
+ async_list->io_pages = 0;
+ }
+ }
+
+ if (async_list->file != filp) {
+ async_list->io_pages = 0;
+ async_list->file = filp;
+ }
+ async_list->io_end = io_end;
+}
+
static ssize_t io_read(struct io_kiocb *req, const struct sqe_submit *s,
bool force_nonblock, struct io_submit_state *state)
{
@@ -806,6 +858,7 @@ static ssize_t io_read(struct io_kiocb *req, const struct sqe_submit *s,
struct kiocb *kiocb = &req->rw;
struct iov_iter iter;
struct file *file;
+ size_t iov_count;
ssize_t ret;
ret = io_prep_rw(req, s->sqe, force_nonblock, state);
@@ -824,16 +877,24 @@ static ssize_t io_read(struct io_kiocb *req, const struct sqe_submit *s,
if (ret)
goto out_fput;
- ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
+ iov_count = iov_iter_count(&iter);
+ ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_count);
if (!ret) {
ssize_t ret2;
/* Catch -EAGAIN return for forced non-blocking submission */
ret2 = call_read_iter(file, kiocb, &iter);
- if (!force_nonblock || ret2 != -EAGAIN)
+ if (!force_nonblock || ret2 != -EAGAIN) {
io_rw_done(kiocb, ret2);
- else
+ } else {
+ /*
+ * If ->needs_lock is true, we're already in async
+ * context.
+ */
+ if (!s->needs_lock)
+ io_async_list_note(READ, req, iov_count);
ret = -EAGAIN;
+ }
}
kfree(iovec);
out_fput:
@@ -849,6 +910,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct sqe_submit *s,
struct kiocb *kiocb = &req->rw;
struct iov_iter iter;
struct file *file;
+ size_t iov_count;
ssize_t ret;
ret = io_prep_rw(req, s->sqe, force_nonblock, state);
@@ -856,10 +918,6 @@ static ssize_t io_write(struct io_kiocb *req, const struct sqe_submit *s,
return ret;
file = kiocb->ki_filp;
- ret = -EAGAIN;
- if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT))
- goto out_fput;
-
ret = -EBADF;
if (unlikely(!(file->f_mode & FMODE_WRITE)))
goto out_fput;
@@ -871,8 +929,17 @@ static ssize_t io_write(struct io_kiocb *req, const struct sqe_submit *s,
if (ret)
goto out_fput;
- ret = rw_verify_area(WRITE, file, &kiocb->ki_pos,
- iov_iter_count(&iter));
+ iov_count = iov_iter_count(&iter);
+
+ ret = -EAGAIN;
+ if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT)) {
+ /* If ->needs_lock is true, we're already in async context. */
+ if (!s->needs_lock)
+ io_async_list_note(WRITE, req, iov_count);
+ goto out_free;
+ }
+
+ ret = rw_verify_area(WRITE, file, &kiocb->ki_pos, iov_count);
if (!ret) {
/*
* Open-code file_start_write here to grab freeze protection,
@@ -890,6 +957,7 @@ static ssize_t io_write(struct io_kiocb *req, const struct sqe_submit *s,
kiocb->ki_flags |= IOCB_WRITE;
io_rw_done(kiocb, call_write_iter(file, kiocb, &iter));
}
+out_free:
kfree(iovec);
out_fput:
if (unlikely(ret))
@@ -1256,6 +1324,21 @@ static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
return 0;
}
+static struct async_list *io_async_list_from_sqe(struct io_ring_ctx *ctx,
+ const struct io_uring_sqe *sqe)
+{
+ switch (sqe->opcode) {
+ case IORING_OP_READV:
+ case IORING_OP_READ_FIXED:
+ return &ctx->pending_async[READ];
+ case IORING_OP_WRITEV:
+ case IORING_OP_WRITE_FIXED:
+ return &ctx->pending_async[WRITE];
+ default:
+ return NULL;
+ }
+}
+
static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
{
u8 opcode = READ_ONCE(sqe->opcode);
@@ -1267,63 +1350,109 @@ static inline bool io_sqe_needs_user(const struct io_uring_sqe *sqe)
static void io_sq_wq_submit_work(struct work_struct *work)
{
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
- struct sqe_submit *s = &req->submit;
struct io_ring_ctx *ctx = req->ctx;
+ struct mm_struct *cur_mm = NULL;
struct files_struct *old_files;
+ struct async_list *async_list;
+ LIST_HEAD(req_list);
mm_segment_t old_fs;
- bool needs_user;
- u64 user_data;
int ret;
- /* Ensure we clear previously set forced non-block flag */
- req->flags &= ~REQ_F_FORCE_NONBLOCK;
-
task_lock(current);
old_files = current->files;
current->files = ctx->sqo_files;
task_unlock(current);
- user_data = READ_ONCE(s->sqe->user_data);
- s->needs_lock = true;
- s->has_user = false;
+ async_list = io_async_list_from_sqe(ctx, req->submit.sqe);
+restart:
+ do {
+ struct sqe_submit *s = &req->submit;
+ u64 user_data = READ_ONCE(s->sqe->user_data);
+
+ /* Ensure we clear previously set forced non-block flag */
+ req->flags &= ~REQ_F_FORCE_NONBLOCK;
+
+ ret = 0;
+ if (io_sqe_needs_user(s->sqe) && !cur_mm) {
+ if (!mmget_not_zero(ctx->sqo_mm)) {
+ ret = -EFAULT;
+ } else {
+ cur_mm = ctx->sqo_mm;;
+ use_mm(ctx->sqo_mm);
+ old_fs = get_fs();
+ set_fs(USER_DS);
+ }
+ }
+
+ if (!ret) {
+ s->has_user = cur_mm != NULL;
+ s->needs_lock = true;
+ do {
+ ret = __io_submit_sqe(ctx, req, s, false, NULL);
+ /*
+ * We can get EAGAIN for polled IO even though
+ * we're forcing a sync submission from here,
+ * since we can't wait for request slots on the
+ * block side.
+ */
+ if (ret != -EAGAIN)
+ break;
+ cond_resched();
+ } while (1);
+ }
+ if (ret) {
+ io_cqring_add_event(ctx, user_data, ret, 0);
+ io_free_req(req);
+ }
+ if (!async_list)
+ break;
+ if (!list_empty(&req_list)) {
+ req = list_first_entry(&req_list, struct io_kiocb,
+ list);
+ list_del(&req->list);
+ continue;
+ }
+ if (list_empty(&async_list->list))
+ break;
+
+ req = NULL;
+ spin_lock(&async_list->lock);
+ if (list_empty(&async_list->list)) {
+ spin_unlock(&async_list->lock);
+ break;
+ }
+ list_splice_init(&async_list->list, &req_list);
+ spin_unlock(&async_list->lock);
+
+ req = list_first_entry(&req_list, struct io_kiocb, list);
+ list_del(&req->list);
+ } while (req);
/*
- * If we're doing IO to fixed buffers, we don't need to get/set
- * user context
+ * Rare case of racing with a submitter. If we find the count has
+ * dropped to zero AND we have pending work items, then restart
+ * the processing. This is a tiny race window.
*/
- needs_user = io_sqe_needs_user(s->sqe);
- if (needs_user) {
- if (!mmget_not_zero(ctx->sqo_mm)) {
- ret = -EFAULT;
- goto err;
+ ret = atomic_dec_return(&async_list->cnt);
+ while (!ret && !list_empty(&async_list->list)) {
+ spin_lock(&async_list->lock);
+ atomic_inc(&async_list->cnt);
+ list_splice_init(&async_list->list, &req_list);
+ spin_unlock(&async_list->lock);
+
+ if (!list_empty(&req_list)) {
+ req = list_first_entry(&req_list, struct io_kiocb,
+ list);
+ list_del(&req->list);
+ goto restart;
}
- use_mm(ctx->sqo_mm);
- old_fs = get_fs();
- set_fs(USER_DS);
- s->has_user = true;
+ ret = atomic_dec_return(&async_list->cnt);
}
- do {
- ret = __io_submit_sqe(ctx, req, s, false, NULL);
- /*
- * We can get EAGAIN for polled IO even though we're forcing
- * a sync submission from here, since we can't wait for
- * request slots on the block side.
- */
- if (ret != -EAGAIN)
- break;
- cond_resched();
- } while (1);
-
- if (needs_user) {
+ if (cur_mm) {
set_fs(old_fs);
- unuse_mm(ctx->sqo_mm);
- mmput(ctx->sqo_mm);
- }
-err:
- if (ret) {
- io_cqring_add_event(ctx, user_data, ret, 0);
- io_free_req(req);
+ unuse_mm(cur_mm);
+ mmput(cur_mm);
}
task_lock(current);
@@ -1331,6 +1460,33 @@ static void io_sq_wq_submit_work(struct work_struct *work)
task_unlock(current);
}
+/*
+ * See if we can piggy back onto previously submitted work, that is still
+ * running. We currently only allow this if the new request is sequential
+ * to the previous one we punted.
+ */
+static bool io_add_to_prev_work(struct async_list *list, struct io_kiocb *req)
+{
+ bool ret = false;
+
+ if (!list)
+ return false;
+ if (!(req->flags & REQ_F_SEQ_PREV))
+ return false;
+ if (!atomic_read(&list->cnt))
+ return false;
+
+ ret = true;
+ spin_lock(&list->lock);
+ list_add_tail(&req->list, &list->list);
+ if (!atomic_read(&list->cnt)) {
+ list_del_init(&req->list);
+ ret = false;
+ }
+ spin_unlock(&list->lock);
+ return ret;
+}
+
static int io_submit_sqe(struct io_ring_ctx *ctx, const struct sqe_submit *s,
struct io_submit_state *state)
{
@@ -1347,9 +1503,16 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, const struct sqe_submit *s,
ret = __io_submit_sqe(ctx, req, s, true, state);
if (ret == -EAGAIN) {
+ struct async_list *list;
+
+ list = io_async_list_from_sqe(ctx, s->sqe);
memcpy(&req->submit, s, sizeof(*s));
- INIT_WORK(&req->work, io_sq_wq_submit_work);
- queue_work(ctx->sqo_wq, &req->work);
+ if (!io_add_to_prev_work(list, req)) {
+ if (list)
+ atomic_inc(&list->cnt);
+ INIT_WORK(&req->work, io_sq_wq_submit_work);
+ queue_work(ctx->sqo_wq, &req->work);
+ }
ret = 0;
}
if (ret)
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* [PATCH 18/18] io_uring: add io_uring_event cache hit information
From: Jens Axboe @ 2019-02-01 15:24 UTC (permalink / raw)
To: linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh, Jens Axboe
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
Add hint on whether a read was served out of the page cache, or if it
hit media. This is useful for buffered async IO, O_DIRECT reads would
never have this set (for obvious reasons).
If the read hit page cache, cqe->flags will have IOCQE_FLAG_CACHEHIT
set.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
fs/io_uring.c | 7 ++++++-
include/uapi/linux/io_uring.h | 5 +++++
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 0e700e81ac43..48234304ef02 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -560,11 +560,16 @@ static void io_fput(struct io_kiocb *req)
static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
{
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw);
+ unsigned ev_flags = 0;
kiocb_end_write(kiocb);
io_fput(req);
- io_cqring_add_event(req->ctx, req->user_data, res, 0);
+
+ if (res > 0 && (req->flags & REQ_F_FORCE_NONBLOCK))
+ ev_flags = IOCQE_FLAG_CACHEHIT;
+
+ io_cqring_add_event(req->ctx, req->user_data, res, ev_flags);
io_free_req(req);
}
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index d319b2cd6319..6782e4e0464b 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -69,6 +69,11 @@ struct io_uring_cqe {
__u32 flags;
};
+/*
+ * io_uring_event->flags
+ */
+#define IOCQE_FLAG_CACHEHIT (1U << 0) /* IO did not hit media */
+
/*
* Magic offsets for the application to mmap the data it needs
*/
--
2.17.1
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* Re: [PATCH 05/18] Add io_uring IO interface
From: Matt Mullins @ 2019-02-01 16:57 UTC (permalink / raw)
To: jannh@google.com, axboe@kernel.dk
Cc: linux-aio@kvack.org, linux-block@vger.kernel.org,
jmoyer@redhat.com, linux-api@vger.kernel.org, hch@lst.de,
viro@zeniv.linux.org.uk, linux-man@vger.kernel.org,
avi@scylladb.com
In-Reply-To: <CAG48ez17NW0GJVRC6dFcHZTgQifFz5og1XCUbXkHKhr6f=j74Q@mail.gmail.com>
[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="utf-8", Size: 13683 bytes --]
On Tue, 2019-01-29 at 00:59 +0100, Jann Horn wrote:
> On Tue, Jan 29, 2019 at 12:47 AM Jens Axboe <axboe@kernel.dk> wrote:
> > On 1/28/19 3:32 PM, Jann Horn wrote:
> > > On Mon, Jan 28, 2019 at 10:35 PM Jens Axboe <axboe@kernel.dk> wrote:
> > > > The submission queue (SQ) and completion queue (CQ) rings are shared
> > > > between the application and the kernel. This eliminates the need to
> > > > copy data back and forth to submit and complete IO.
> > > >
> > > > IO submissions use the io_uring_sqe data structure, and completions
> > > > are generated in the form of io_uring_sqe data structures. The SQ
> > > > ring is an index into the io_uring_sqe array, which makes it possible
> > > > to submit a batch of IOs without them being contiguous in the ring.
> > > > The CQ ring is always contiguous, as completion events are inherently
> > > > unordered, and hence any io_uring_cqe entry can point back to an
> > > > arbitrary submission.
> > > >
> > > > Two new system calls are added for this:
> > > >
> > > > io_uring_setup(entries, params)
> > > > Sets up a context for doing async IO. On success, returns a file
> > > > descriptor that the application can mmap to gain access to the
> > > > SQ ring, CQ ring, and io_uring_sqes.
> > > >
> > > > io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
> > > > Initiates IO against the rings mapped to this fd, or waits for
> > > > them to complete, or both. The behavior is controlled by the
> > > > parameters passed in. If 'to_submit' is non-zero, then we'll
> > > > try and submit new IO. If IORING_ENTER_GETEVENTS is set, the
> > > > kernel will wait for 'min_complete' events, if they aren't
> > > > already available. It's valid to set IORING_ENTER_GETEVENTS
> > > > and 'min_complete' == 0 at the same time, this allows the
> > > > kernel to return already completed events without waiting
> > > > for them. This is useful only for polling, as for IRQ
> > > > driven IO, the application can just check the CQ ring
> > > > without entering the kernel.
> > > >
> > > > With this setup, it's possible to do async IO with a single system
> > > > call. Future developments will enable polled IO with this interface,
> > > > and polled submission as well. The latter will enable an application
> > > > to do IO without doing ANY system calls at all.
> > > >
> > > > For IRQ driven IO, an application only needs to enter the kernel for
> > > > completions if it wants to wait for them to occur.
> > > >
> > > > Each io_uring is backed by a workqueue, to support buffered async IO
> > > > as well. We will only punt to an async context if the command would
> > > > need to wait for IO on the device side. Any data that can be accessed
> > > > directly in the page cache is done inline. This avoids the slowness
> > > > issue of usual threadpools, since cached data is accessed as quickly
> > > > as a sync interface.
> > > >
> > > > Sample application: https://urldefense.proofpoint.com/v2/url?u=http-3A__git.kernel.dk_cgit_fio_plain_t_io-5Furing.c&d=DwIBaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=pqM-eO4A2hNFhIFiX-7eGg&m=MGr14pOzNbC7Z-8_dV4GMiH3AbkkH0RSQoQ894Tu0yc&s=mgbcubzOMiCpFpnwW-HA3ey0YDYPkgMIZ7Bmy4w6Chc&e=
> > >
> > > [...]
> > > > +static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
> > > > + bool force_nonblock)
> > > > +{
> > > > + struct kiocb *kiocb = &req->rw;
> > > > + int ret;
> > > > +
> > > > + kiocb->ki_filp = fget(sqe->fd);
> > > > + if (unlikely(!kiocb->ki_filp))
> > > > + return -EBADF;
> > > > + kiocb->ki_pos = sqe->off;
> > > > + kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
> > > > + kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
> > > > + if (sqe->ioprio) {
> > > > + ret = ioprio_check_cap(sqe->ioprio);
> > > > + if (ret)
> > > > + goto out_fput;
> > > > +
> > > > + kiocb->ki_ioprio = sqe->ioprio;
> > > > + } else
> > > > + kiocb->ki_ioprio = get_current_ioprio();
> > > > +
> > > > + ret = kiocb_set_rw_flags(kiocb, sqe->rw_flags);
> > > > + if (unlikely(ret))
> > > > + goto out_fput;
> > > > + if (force_nonblock) {
> > > > + kiocb->ki_flags |= IOCB_NOWAIT;
> > > > + req->flags |= REQ_F_FORCE_NONBLOCK;
> > > > + }
> > > > + if (kiocb->ki_flags & IOCB_HIPRI) {
> > > > + ret = -EINVAL;
> > > > + goto out_fput;
> > > > + }
> > > > +
> > > > + kiocb->ki_complete = io_complete_rw;
> > > > + return 0;
> > > > +out_fput:
> > > > + fput(kiocb->ki_filp);
> > > > + return ret;
> > > > +}
> > >
> > > [...]
> > > > +static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
> > > > + bool force_nonblock)
> > > > +{
> > > > + struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
> > > > + struct kiocb *kiocb = &req->rw;
> > > > + struct iov_iter iter;
> > > > + struct file *file;
> > > > + ssize_t ret;
> > > > +
> > > > + ret = io_prep_rw(req, sqe, force_nonblock);
> > > > + if (ret)
> > > > + return ret;
> > > > + file = kiocb->ki_filp;
> > > > +
> > > > + ret = -EBADF;
> > > > + if (unlikely(!(file->f_mode & FMODE_READ)))
> > > > + goto out_fput;
> > > > + ret = -EINVAL;
> > > > + if (unlikely(!file->f_op->read_iter))
> > > > + goto out_fput;
> > > > +
> > > > + ret = io_import_iovec(req->ctx, READ, sqe, &iovec, &iter);
> > > > + if (ret)
> > > > + goto out_fput;
> > > > +
> > > > + ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
> > > > + if (!ret) {
> > > > + ssize_t ret2;
> > > > +
> > > > + /* Catch -EAGAIN return for forced non-blocking submission */
> > > > + ret2 = call_read_iter(file, kiocb, &iter);
> > > > + if (!force_nonblock || ret2 != -EAGAIN)
> > > > + io_rw_done(kiocb, ret2);
> > > > + else
> > > > + ret = -EAGAIN;
> > > > + }
> > > > + kfree(iovec);
> > > > +out_fput:
> > > > + if (unlikely(ret))
> > > > + fput(file);
> > > > + return ret;
> > > > +}
> > >
> > > [...]
> > > > +static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
> > > > + struct sqe_submit *s, bool force_nonblock)
> > > > +{
> > > > + const struct io_uring_sqe *sqe = s->sqe;
> > > > + ssize_t ret;
> > > > +
> > > > + if (unlikely(s->index >= ctx->sq_entries))
> > > > + return -EINVAL;
> > > > + req->user_data = sqe->user_data;
> > > > +
> > > > + ret = -EINVAL;
> > > > + switch (sqe->opcode) {
> > > > + case IORING_OP_NOP:
> > > > + ret = io_nop(req, sqe);
> > > > + break;
> > > > + case IORING_OP_READV:
> > > > + ret = io_read(req, sqe, force_nonblock);
> > > > + break;
> > > > + case IORING_OP_WRITEV:
> > > > + ret = io_write(req, sqe, force_nonblock);
> > > > + break;
> > > > + default:
> > > > + ret = -EINVAL;
> > > > + break;
> > > > + }
> > > > +
> > > > + return ret;
> > > > +}
> > > > +
> > > > +static void io_sq_wq_submit_work(struct work_struct *work)
> > > > +{
> > > > + struct io_kiocb *req = container_of(work, struct io_kiocb, work);
> > > > + struct sqe_submit *s = &req->submit;
> > > > + u64 user_data = s->sqe->user_data;
> > > > + struct io_ring_ctx *ctx = req->ctx;
> > > > + mm_segment_t old_fs = get_fs();
> > > > + struct files_struct *old_files;
> > > > + int ret;
> > > > +
> > > > + /* Ensure we clear previously set forced non-block flag */
> > > > + req->flags &= ~REQ_F_FORCE_NONBLOCK;
> > > > +
> > > > + old_files = current->files;
> > > > + current->files = ctx->sqo_files;
> > >
> > > I think you're not supposed to twiddle with current->files without
> > > holding task_lock(current).
> >
> > 'current' is the work queue item in this case, do we need to protect
> > against anything else? I can add the locking around the assignments
> > (both places).
>
> Stuff like proc_fd_link() uses get_files_struct(), which grabs a
> reference to your current files_struct protected only by task_lock();
> and it doesn't use anything like READ_ONCE(), so even if the object
> lifetime is not a problem, get_files_struct() could potentially crash
> due to a double-read (reading task->files twice and assuming that the
> result will be the same). As far as I can tell, this procfs code also
> works on kernel threads.
>
> > > > + if (!mmget_not_zero(ctx->sqo_mm)) {
> > > > + ret = -EFAULT;
> > > > + goto err;
> > > > + }
> > > > +
> > > > + use_mm(ctx->sqo_mm);
> > > > + set_fs(USER_DS);
> > > > +
> > > > + ret = __io_submit_sqe(ctx, req, s, false);
> > > > +
> > > > + set_fs(old_fs);
> > > > + unuse_mm(ctx->sqo_mm);
> > > > + mmput(ctx->sqo_mm);
> > > > +err:
> > > > + if (ret) {
> > > > + io_cqring_add_event(ctx, user_data, ret, 0);
> > > > + io_free_req(req);
> > > > + }
> > > > + current->files = old_files;
> > > > +}
> > >
> > > [...]
> > > > +static int io_sq_offload_start(struct io_ring_ctx *ctx)
> > > > +{
> > > > + int ret;
> > > > +
> > > > + ctx->sqo_mm = current->mm;
> > >
> > > What keeps this thing alive?
> >
> > I think we're deadling with the same thing as the files below, I'll
> > defer to that.
> >
> > > > + /*
> > > > + * This is safe since 'current' has the fd installed, and if that gets
> > > > + * closed on exit, then fops->release() is invoked which waits for the
> > > > + * async contexts to flush and exit before exiting.
> > > > + */
> > > > + ret = -EBADF;
> > > > + ctx->sqo_files = current->files;
> > > > + if (!ctx->sqo_files)
> > > > + goto err;
> > >
> > > That's gnarly. Adding Al Viro to the thread.
> > >
> > > I think you misunderstand the semantics of f_op->release. The ->flush
> > > handler is invoked whenever a file descriptor is closed through
> > > filp_close() (via deletion of the files_struct, sys_close(),
> > > sys_dup2(), ...), so if you had used that one, _maybe_ this would
> > > work. But the ->release handler only runs when the _last_ reference to
> > > a struct file has been dropped - so you can, for example, fork() a
> > > child, then exit() in the parent, and the ->release handler isn't
> > > invoked. So I don't see how this can work.
> >
> > The anonfd is CLOEXEC. The idea is exactly that it only runs when the
> > last reference to the file has been dropped. Not sure why you think I
> > need ->flush() here?
>
> Can't I just use fcntl(fd, F_SETFD, fd, 0) to clear the CLOEXEC flag?
> Or send the fd via SCM_RIGHTS?
>
> > > But even if you had abused ->flush for this instead: close_files()
> > > currently has a comment in it that claims that "this is the last
> > > reference to the files structure"; this change would make that claim
> > > untrue.
> >
> > Let me see if I can explain my intent better than that comment... We
> > know the parent who set up the io_uring instance will be around for as
> > long as io_uring instance persists.
>
> That's the part that I think is wrong: As far as I can tell, the
> parent can go away and you won't notice.
>
> Also, note that "the parent" is different things for ->files and ->mm.
> You can have a multithreaded process whose threads don't have the same
> ->files, or multiple process that share ->files without sharing ->mm,
> ...
This had actually been get_files_struct() in early versions, and I had
reported to Jens that it allows something like
int main() {
struct io_uring_params uring_params = {
.flags = IORING_SETUP_SQPOLL,
};
int uring_fd = syscall(425 /* io_uring_setup */, 16, &uring_params);
}
to leak both the files_struct and the kthread, as the files_struct and
the uring context form a circular reference. I haven't really come up
with a good way to reconcile the requirements here; perhaps we need an
exit_uring() akin to exit_aio()?
> > When we are tearing down the
> > io_uring, then we wait for any async contexts (like the one above) to
> > exit. Once they are exited, it's safe to proceed with the exit and
> > teardown ->files[].
>
> But you only do that teardown on ->release, right? And ->release
> doesn't have much to do with the process lifetime.
>
> > That's the idea... Not trying to be clever, some of this dates back to
> > the aio weirdness where it was impossible to have cross references like
> > this, since it would lead to teardown deadlocks with how exit_aio() is
> > used. I can probably grab a struct files reference above, but currently
> > I don't see why it's needed.
N‹§²æìr¸›zǧu©ž²Æ {\b†éì¹»\x1c®&Þ–)îŨ¨Š{ayº\x1dÊÚ&j:+v‰¨’öœ’Šà\x16Šæ¢·¢ú(œ¸§»\x10\b:Çž†Ûiÿü0ÂKÚrJ+ƒö¢£ðèž×¦j)Z†·Ÿ
^ permalink raw reply
* Re: [PATCH 04/18] iomap: wire up the iopoll method
From: Bart Van Assche @ 2019-02-01 16:59 UTC (permalink / raw)
To: Jens Axboe, linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh
In-Reply-To: <20190201152414.23296-5-axboe@kernel.dk>
On Fri, 2019-02-01 at 08:24 -0700, Jens Axboe wrote:
> +int iomap_dio_iopoll(struct kiocb *kiocb, bool spin)
> +{
> + struct request_queue *q = READ_ONCE(kiocb->private);
> +
> + if (!q)
> + return 0;
> + return blk_poll(q, READ_ONCE(kiocb->ki_cookie), spin);
> +}
> +EXPORT_SYMBOL_GPL(iomap_dio_iopoll);
How does this interact with block device removal? My understanding is that this
function can get called after a request has completed. Which mechanism, if any,
prevents that request queue 'q' is removed after kiocb->private has been read
and before blk_poll() is called?
Thanks,
Bart.
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: [PATCH 05/18] Add io_uring IO interface
From: Jann Horn @ 2019-02-01 17:04 UTC (permalink / raw)
To: Matt Mullins
Cc: axboe@kernel.dk, linux-aio@kvack.org, linux-block@vger.kernel.org,
jmoyer@redhat.com, linux-api@vger.kernel.org, hch@lst.de,
viro@zeniv.linux.org.uk, linux-man@vger.kernel.org,
avi@scylladb.com
In-Reply-To: <05cb18f7a97a6151c305cdb7240c4abc995aed59.camel@fb.com>
On Fri, Feb 1, 2019 at 5:57 PM Matt Mullins <mmullins@fb.com> wrote:
> On Tue, 2019-01-29 at 00:59 +0100, Jann Horn wrote:
> > On Tue, Jan 29, 2019 at 12:47 AM Jens Axboe <axboe@kernel.dk> wrote:
> > > On 1/28/19 3:32 PM, Jann Horn wrote:
> > > > On Mon, Jan 28, 2019 at 10:35 PM Jens Axboe <axboe@kernel.dk> wrote:
> > > > > The submission queue (SQ) and completion queue (CQ) rings are shared
> > > > > between the application and the kernel. This eliminates the need to
> > > > > copy data back and forth to submit and complete IO.
> > > > >
> > > > > IO submissions use the io_uring_sqe data structure, and completions
> > > > > are generated in the form of io_uring_sqe data structures. The SQ
> > > > > ring is an index into the io_uring_sqe array, which makes it possible
> > > > > to submit a batch of IOs without them being contiguous in the ring.
> > > > > The CQ ring is always contiguous, as completion events are inherently
> > > > > unordered, and hence any io_uring_cqe entry can point back to an
> > > > > arbitrary submission.
> > > > >
> > > > > Two new system calls are added for this:
> > > > >
> > > > > io_uring_setup(entries, params)
> > > > > Sets up a context for doing async IO. On success, returns a file
> > > > > descriptor that the application can mmap to gain access to the
> > > > > SQ ring, CQ ring, and io_uring_sqes.
> > > > >
> > > > > io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
> > > > > Initiates IO against the rings mapped to this fd, or waits for
> > > > > them to complete, or both. The behavior is controlled by the
> > > > > parameters passed in. If 'to_submit' is non-zero, then we'll
> > > > > try and submit new IO. If IORING_ENTER_GETEVENTS is set, the
> > > > > kernel will wait for 'min_complete' events, if they aren't
> > > > > already available. It's valid to set IORING_ENTER_GETEVENTS
> > > > > and 'min_complete' == 0 at the same time, this allows the
> > > > > kernel to return already completed events without waiting
> > > > > for them. This is useful only for polling, as for IRQ
> > > > > driven IO, the application can just check the CQ ring
> > > > > without entering the kernel.
> > > > >
> > > > > With this setup, it's possible to do async IO with a single system
> > > > > call. Future developments will enable polled IO with this interface,
> > > > > and polled submission as well. The latter will enable an application
> > > > > to do IO without doing ANY system calls at all.
> > > > >
> > > > > For IRQ driven IO, an application only needs to enter the kernel for
> > > > > completions if it wants to wait for them to occur.
> > > > >
> > > > > Each io_uring is backed by a workqueue, to support buffered async IO
> > > > > as well. We will only punt to an async context if the command would
> > > > > need to wait for IO on the device side. Any data that can be accessed
> > > > > directly in the page cache is done inline. This avoids the slowness
> > > > > issue of usual threadpools, since cached data is accessed as quickly
> > > > > as a sync interface.
> > > > >
> > > > > Sample application: https://urldefense.proofpoint.com/v2/url?u=http-3A__git.kernel.dk_cgit_fio_plain_t_io-5Furing.c&d=DwIBaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=pqM-eO4A2hNFhIFiX-7eGg&m=MGr14pOzNbC7Z-8_dV4GMiH3AbkkH0RSQoQ894Tu0yc&s=mgbcubzOMiCpFpnwW-HA3ey0YDYPkgMIZ7Bmy4w6Chc&e=
> > > >
> > > > [...]
> > > > > +static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
> > > > > + bool force_nonblock)
> > > > > +{
> > > > > + struct kiocb *kiocb = &req->rw;
> > > > > + int ret;
> > > > > +
> > > > > + kiocb->ki_filp = fget(sqe->fd);
> > > > > + if (unlikely(!kiocb->ki_filp))
> > > > > + return -EBADF;
> > > > > + kiocb->ki_pos = sqe->off;
> > > > > + kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
> > > > > + kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
> > > > > + if (sqe->ioprio) {
> > > > > + ret = ioprio_check_cap(sqe->ioprio);
> > > > > + if (ret)
> > > > > + goto out_fput;
> > > > > +
> > > > > + kiocb->ki_ioprio = sqe->ioprio;
> > > > > + } else
> > > > > + kiocb->ki_ioprio = get_current_ioprio();
> > > > > +
> > > > > + ret = kiocb_set_rw_flags(kiocb, sqe->rw_flags);
> > > > > + if (unlikely(ret))
> > > > > + goto out_fput;
> > > > > + if (force_nonblock) {
> > > > > + kiocb->ki_flags |= IOCB_NOWAIT;
> > > > > + req->flags |= REQ_F_FORCE_NONBLOCK;
> > > > > + }
> > > > > + if (kiocb->ki_flags & IOCB_HIPRI) {
> > > > > + ret = -EINVAL;
> > > > > + goto out_fput;
> > > > > + }
> > > > > +
> > > > > + kiocb->ki_complete = io_complete_rw;
> > > > > + return 0;
> > > > > +out_fput:
> > > > > + fput(kiocb->ki_filp);
> > > > > + return ret;
> > > > > +}
> > > >
> > > > [...]
> > > > > +static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
> > > > > + bool force_nonblock)
> > > > > +{
> > > > > + struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
> > > > > + struct kiocb *kiocb = &req->rw;
> > > > > + struct iov_iter iter;
> > > > > + struct file *file;
> > > > > + ssize_t ret;
> > > > > +
> > > > > + ret = io_prep_rw(req, sqe, force_nonblock);
> > > > > + if (ret)
> > > > > + return ret;
> > > > > + file = kiocb->ki_filp;
> > > > > +
> > > > > + ret = -EBADF;
> > > > > + if (unlikely(!(file->f_mode & FMODE_READ)))
> > > > > + goto out_fput;
> > > > > + ret = -EINVAL;
> > > > > + if (unlikely(!file->f_op->read_iter))
> > > > > + goto out_fput;
> > > > > +
> > > > > + ret = io_import_iovec(req->ctx, READ, sqe, &iovec, &iter);
> > > > > + if (ret)
> > > > > + goto out_fput;
> > > > > +
> > > > > + ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
> > > > > + if (!ret) {
> > > > > + ssize_t ret2;
> > > > > +
> > > > > + /* Catch -EAGAIN return for forced non-blocking submission */
> > > > > + ret2 = call_read_iter(file, kiocb, &iter);
> > > > > + if (!force_nonblock || ret2 != -EAGAIN)
> > > > > + io_rw_done(kiocb, ret2);
> > > > > + else
> > > > > + ret = -EAGAIN;
> > > > > + }
> > > > > + kfree(iovec);
> > > > > +out_fput:
> > > > > + if (unlikely(ret))
> > > > > + fput(file);
> > > > > + return ret;
> > > > > +}
> > > >
> > > > [...]
> > > > > +static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
> > > > > + struct sqe_submit *s, bool force_nonblock)
> > > > > +{
> > > > > + const struct io_uring_sqe *sqe = s->sqe;
> > > > > + ssize_t ret;
> > > > > +
> > > > > + if (unlikely(s->index >= ctx->sq_entries))
> > > > > + return -EINVAL;
> > > > > + req->user_data = sqe->user_data;
> > > > > +
> > > > > + ret = -EINVAL;
> > > > > + switch (sqe->opcode) {
> > > > > + case IORING_OP_NOP:
> > > > > + ret = io_nop(req, sqe);
> > > > > + break;
> > > > > + case IORING_OP_READV:
> > > > > + ret = io_read(req, sqe, force_nonblock);
> > > > > + break;
> > > > > + case IORING_OP_WRITEV:
> > > > > + ret = io_write(req, sqe, force_nonblock);
> > > > > + break;
> > > > > + default:
> > > > > + ret = -EINVAL;
> > > > > + break;
> > > > > + }
> > > > > +
> > > > > + return ret;
> > > > > +}
> > > > > +
> > > > > +static void io_sq_wq_submit_work(struct work_struct *work)
> > > > > +{
> > > > > + struct io_kiocb *req = container_of(work, struct io_kiocb, work);
> > > > > + struct sqe_submit *s = &req->submit;
> > > > > + u64 user_data = s->sqe->user_data;
> > > > > + struct io_ring_ctx *ctx = req->ctx;
> > > > > + mm_segment_t old_fs = get_fs();
> > > > > + struct files_struct *old_files;
> > > > > + int ret;
> > > > > +
> > > > > + /* Ensure we clear previously set forced non-block flag */
> > > > > + req->flags &= ~REQ_F_FORCE_NONBLOCK;
> > > > > +
> > > > > + old_files = current->files;
> > > > > + current->files = ctx->sqo_files;
> > > >
> > > > I think you're not supposed to twiddle with current->files without
> > > > holding task_lock(current).
> > >
> > > 'current' is the work queue item in this case, do we need to protect
> > > against anything else? I can add the locking around the assignments
> > > (both places).
> >
> > Stuff like proc_fd_link() uses get_files_struct(), which grabs a
> > reference to your current files_struct protected only by task_lock();
> > and it doesn't use anything like READ_ONCE(), so even if the object
> > lifetime is not a problem, get_files_struct() could potentially crash
> > due to a double-read (reading task->files twice and assuming that the
> > result will be the same). As far as I can tell, this procfs code also
> > works on kernel threads.
> >
> > > > > + if (!mmget_not_zero(ctx->sqo_mm)) {
> > > > > + ret = -EFAULT;
> > > > > + goto err;
> > > > > + }
> > > > > +
> > > > > + use_mm(ctx->sqo_mm);
> > > > > + set_fs(USER_DS);
> > > > > +
> > > > > + ret = __io_submit_sqe(ctx, req, s, false);
> > > > > +
> > > > > + set_fs(old_fs);
> > > > > + unuse_mm(ctx->sqo_mm);
> > > > > + mmput(ctx->sqo_mm);
> > > > > +err:
> > > > > + if (ret) {
> > > > > + io_cqring_add_event(ctx, user_data, ret, 0);
> > > > > + io_free_req(req);
> > > > > + }
> > > > > + current->files = old_files;
> > > > > +}
> > > >
> > > > [...]
> > > > > +static int io_sq_offload_start(struct io_ring_ctx *ctx)
> > > > > +{
> > > > > + int ret;
> > > > > +
> > > > > + ctx->sqo_mm = current->mm;
> > > >
> > > > What keeps this thing alive?
> > >
> > > I think we're deadling with the same thing as the files below, I'll
> > > defer to that.
> > >
> > > > > + /*
> > > > > + * This is safe since 'current' has the fd installed, and if that gets
> > > > > + * closed on exit, then fops->release() is invoked which waits for the
> > > > > + * async contexts to flush and exit before exiting.
> > > > > + */
> > > > > + ret = -EBADF;
> > > > > + ctx->sqo_files = current->files;
> > > > > + if (!ctx->sqo_files)
> > > > > + goto err;
> > > >
> > > > That's gnarly. Adding Al Viro to the thread.
> > > >
> > > > I think you misunderstand the semantics of f_op->release. The ->flush
> > > > handler is invoked whenever a file descriptor is closed through
> > > > filp_close() (via deletion of the files_struct, sys_close(),
> > > > sys_dup2(), ...), so if you had used that one, _maybe_ this would
> > > > work. But the ->release handler only runs when the _last_ reference to
> > > > a struct file has been dropped - so you can, for example, fork() a
> > > > child, then exit() in the parent, and the ->release handler isn't
> > > > invoked. So I don't see how this can work.
> > >
> > > The anonfd is CLOEXEC. The idea is exactly that it only runs when the
> > > last reference to the file has been dropped. Not sure why you think I
> > > need ->flush() here?
> >
> > Can't I just use fcntl(fd, F_SETFD, fd, 0) to clear the CLOEXEC flag?
> > Or send the fd via SCM_RIGHTS?
> >
> > > > But even if you had abused ->flush for this instead: close_files()
> > > > currently has a comment in it that claims that "this is the last
> > > > reference to the files structure"; this change would make that claim
> > > > untrue.
> > >
> > > Let me see if I can explain my intent better than that comment... We
> > > know the parent who set up the io_uring instance will be around for as
> > > long as io_uring instance persists.
> >
> > That's the part that I think is wrong: As far as I can tell, the
> > parent can go away and you won't notice.
> >
> > Also, note that "the parent" is different things for ->files and ->mm.
> > You can have a multithreaded process whose threads don't have the same
> > ->files, or multiple process that share ->files without sharing ->mm,
> > ...
>
> This had actually been get_files_struct() in early versions, and I had
> reported to Jens that it allows something like
>
> int main() {
> struct io_uring_params uring_params = {
> .flags = IORING_SETUP_SQPOLL,
> };
> int uring_fd = syscall(425 /* io_uring_setup */, 16, &uring_params);
> }
>
> to leak both the files_struct and the kthread, as the files_struct and
> the uring context form a circular reference. I haven't really come up
> with a good way to reconcile the requirements here; perhaps we need an
> exit_uring() akin to exit_aio()?
Oh, yuck. Uuuh... can we make "struct files_struct" doubly-refcounted,
like "struct mm_struct"? One reference type to keep the contents
intact (the reference type you normally use, and the type used by
uring when the thread is running), and one reference type to just keep
the struct itself existing, but without preserving its contents
(reference held consistently by the uring thread)?
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: [PATCH 05/18] Add io_uring IO interface
From: Jann Horn @ 2019-02-01 17:23 UTC (permalink / raw)
To: Matt Mullins, viro@zeniv.linux.org.uk, axboe@kernel.dk,
linux-fsdevel
Cc: linux-aio@kvack.org, linux-block@vger.kernel.org,
jmoyer@redhat.com, linux-api@vger.kernel.org, hch@lst.de,
linux-man@vger.kernel.org, avi@scylladb.com
In-Reply-To: <CAG48ez09iOOnPz83b6HxktYHTfouS2GD6i3PfQEjp8WCp+3-VA@mail.gmail.com>
On Fri, Feb 1, 2019 at 6:04 PM Jann Horn <jannh@google.com> wrote:
>
> On Fri, Feb 1, 2019 at 5:57 PM Matt Mullins <mmullins@fb.com> wrote:
> > On Tue, 2019-01-29 at 00:59 +0100, Jann Horn wrote:
> > > On Tue, Jan 29, 2019 at 12:47 AM Jens Axboe <axboe@kernel.dk> wrote:
> > > > On 1/28/19 3:32 PM, Jann Horn wrote:
> > > > > On Mon, Jan 28, 2019 at 10:35 PM Jens Axboe <axboe@kernel.dk> wrote:
> > > > > > The submission queue (SQ) and completion queue (CQ) rings are shared
> > > > > > between the application and the kernel. This eliminates the need to
> > > > > > copy data back and forth to submit and complete IO.
> > > > > >
> > > > > > IO submissions use the io_uring_sqe data structure, and completions
> > > > > > are generated in the form of io_uring_sqe data structures. The SQ
> > > > > > ring is an index into the io_uring_sqe array, which makes it possible
> > > > > > to submit a batch of IOs without them being contiguous in the ring.
> > > > > > The CQ ring is always contiguous, as completion events are inherently
> > > > > > unordered, and hence any io_uring_cqe entry can point back to an
> > > > > > arbitrary submission.
> > > > > >
> > > > > > Two new system calls are added for this:
> > > > > >
> > > > > > io_uring_setup(entries, params)
> > > > > > Sets up a context for doing async IO. On success, returns a file
> > > > > > descriptor that the application can mmap to gain access to the
> > > > > > SQ ring, CQ ring, and io_uring_sqes.
> > > > > >
> > > > > > io_uring_enter(fd, to_submit, min_complete, flags, sigset, sigsetsize)
> > > > > > Initiates IO against the rings mapped to this fd, or waits for
> > > > > > them to complete, or both. The behavior is controlled by the
> > > > > > parameters passed in. If 'to_submit' is non-zero, then we'll
> > > > > > try and submit new IO. If IORING_ENTER_GETEVENTS is set, the
> > > > > > kernel will wait for 'min_complete' events, if they aren't
> > > > > > already available. It's valid to set IORING_ENTER_GETEVENTS
> > > > > > and 'min_complete' == 0 at the same time, this allows the
> > > > > > kernel to return already completed events without waiting
> > > > > > for them. This is useful only for polling, as for IRQ
> > > > > > driven IO, the application can just check the CQ ring
> > > > > > without entering the kernel.
> > > > > >
> > > > > > With this setup, it's possible to do async IO with a single system
> > > > > > call. Future developments will enable polled IO with this interface,
> > > > > > and polled submission as well. The latter will enable an application
> > > > > > to do IO without doing ANY system calls at all.
> > > > > >
> > > > > > For IRQ driven IO, an application only needs to enter the kernel for
> > > > > > completions if it wants to wait for them to occur.
> > > > > >
> > > > > > Each io_uring is backed by a workqueue, to support buffered async IO
> > > > > > as well. We will only punt to an async context if the command would
> > > > > > need to wait for IO on the device side. Any data that can be accessed
> > > > > > directly in the page cache is done inline. This avoids the slowness
> > > > > > issue of usual threadpools, since cached data is accessed as quickly
> > > > > > as a sync interface.
> > > > > >
> > > > > > Sample application: https://urldefense.proofpoint.com/v2/url?u=http-3A__git.kernel.dk_cgit_fio_plain_t_io-5Furing.c&d=DwIBaQ&c=5VD0RTtNlTh3ycd41b3MUw&r=pqM-eO4A2hNFhIFiX-7eGg&m=MGr14pOzNbC7Z-8_dV4GMiH3AbkkH0RSQoQ894Tu0yc&s=mgbcubzOMiCpFpnwW-HA3ey0YDYPkgMIZ7Bmy4w6Chc&e=
> > > > >
> > > > > [...]
> > > > > > +static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
> > > > > > + bool force_nonblock)
> > > > > > +{
> > > > > > + struct kiocb *kiocb = &req->rw;
> > > > > > + int ret;
> > > > > > +
> > > > > > + kiocb->ki_filp = fget(sqe->fd);
> > > > > > + if (unlikely(!kiocb->ki_filp))
> > > > > > + return -EBADF;
> > > > > > + kiocb->ki_pos = sqe->off;
> > > > > > + kiocb->ki_flags = iocb_flags(kiocb->ki_filp);
> > > > > > + kiocb->ki_hint = ki_hint_validate(file_write_hint(kiocb->ki_filp));
> > > > > > + if (sqe->ioprio) {
> > > > > > + ret = ioprio_check_cap(sqe->ioprio);
> > > > > > + if (ret)
> > > > > > + goto out_fput;
> > > > > > +
> > > > > > + kiocb->ki_ioprio = sqe->ioprio;
> > > > > > + } else
> > > > > > + kiocb->ki_ioprio = get_current_ioprio();
> > > > > > +
> > > > > > + ret = kiocb_set_rw_flags(kiocb, sqe->rw_flags);
> > > > > > + if (unlikely(ret))
> > > > > > + goto out_fput;
> > > > > > + if (force_nonblock) {
> > > > > > + kiocb->ki_flags |= IOCB_NOWAIT;
> > > > > > + req->flags |= REQ_F_FORCE_NONBLOCK;
> > > > > > + }
> > > > > > + if (kiocb->ki_flags & IOCB_HIPRI) {
> > > > > > + ret = -EINVAL;
> > > > > > + goto out_fput;
> > > > > > + }
> > > > > > +
> > > > > > + kiocb->ki_complete = io_complete_rw;
> > > > > > + return 0;
> > > > > > +out_fput:
> > > > > > + fput(kiocb->ki_filp);
> > > > > > + return ret;
> > > > > > +}
> > > > >
> > > > > [...]
> > > > > > +static ssize_t io_read(struct io_kiocb *req, const struct io_uring_sqe *sqe,
> > > > > > + bool force_nonblock)
> > > > > > +{
> > > > > > + struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
> > > > > > + struct kiocb *kiocb = &req->rw;
> > > > > > + struct iov_iter iter;
> > > > > > + struct file *file;
> > > > > > + ssize_t ret;
> > > > > > +
> > > > > > + ret = io_prep_rw(req, sqe, force_nonblock);
> > > > > > + if (ret)
> > > > > > + return ret;
> > > > > > + file = kiocb->ki_filp;
> > > > > > +
> > > > > > + ret = -EBADF;
> > > > > > + if (unlikely(!(file->f_mode & FMODE_READ)))
> > > > > > + goto out_fput;
> > > > > > + ret = -EINVAL;
> > > > > > + if (unlikely(!file->f_op->read_iter))
> > > > > > + goto out_fput;
> > > > > > +
> > > > > > + ret = io_import_iovec(req->ctx, READ, sqe, &iovec, &iter);
> > > > > > + if (ret)
> > > > > > + goto out_fput;
> > > > > > +
> > > > > > + ret = rw_verify_area(READ, file, &kiocb->ki_pos, iov_iter_count(&iter));
> > > > > > + if (!ret) {
> > > > > > + ssize_t ret2;
> > > > > > +
> > > > > > + /* Catch -EAGAIN return for forced non-blocking submission */
> > > > > > + ret2 = call_read_iter(file, kiocb, &iter);
> > > > > > + if (!force_nonblock || ret2 != -EAGAIN)
> > > > > > + io_rw_done(kiocb, ret2);
> > > > > > + else
> > > > > > + ret = -EAGAIN;
> > > > > > + }
> > > > > > + kfree(iovec);
> > > > > > +out_fput:
> > > > > > + if (unlikely(ret))
> > > > > > + fput(file);
> > > > > > + return ret;
> > > > > > +}
> > > > >
> > > > > [...]
> > > > > > +static int __io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
> > > > > > + struct sqe_submit *s, bool force_nonblock)
> > > > > > +{
> > > > > > + const struct io_uring_sqe *sqe = s->sqe;
> > > > > > + ssize_t ret;
> > > > > > +
> > > > > > + if (unlikely(s->index >= ctx->sq_entries))
> > > > > > + return -EINVAL;
> > > > > > + req->user_data = sqe->user_data;
> > > > > > +
> > > > > > + ret = -EINVAL;
> > > > > > + switch (sqe->opcode) {
> > > > > > + case IORING_OP_NOP:
> > > > > > + ret = io_nop(req, sqe);
> > > > > > + break;
> > > > > > + case IORING_OP_READV:
> > > > > > + ret = io_read(req, sqe, force_nonblock);
> > > > > > + break;
> > > > > > + case IORING_OP_WRITEV:
> > > > > > + ret = io_write(req, sqe, force_nonblock);
> > > > > > + break;
> > > > > > + default:
> > > > > > + ret = -EINVAL;
> > > > > > + break;
> > > > > > + }
> > > > > > +
> > > > > > + return ret;
> > > > > > +}
> > > > > > +
> > > > > > +static void io_sq_wq_submit_work(struct work_struct *work)
> > > > > > +{
> > > > > > + struct io_kiocb *req = container_of(work, struct io_kiocb, work);
> > > > > > + struct sqe_submit *s = &req->submit;
> > > > > > + u64 user_data = s->sqe->user_data;
> > > > > > + struct io_ring_ctx *ctx = req->ctx;
> > > > > > + mm_segment_t old_fs = get_fs();
> > > > > > + struct files_struct *old_files;
> > > > > > + int ret;
> > > > > > +
> > > > > > + /* Ensure we clear previously set forced non-block flag */
> > > > > > + req->flags &= ~REQ_F_FORCE_NONBLOCK;
> > > > > > +
> > > > > > + old_files = current->files;
> > > > > > + current->files = ctx->sqo_files;
> > > > >
> > > > > I think you're not supposed to twiddle with current->files without
> > > > > holding task_lock(current).
> > > >
> > > > 'current' is the work queue item in this case, do we need to protect
> > > > against anything else? I can add the locking around the assignments
> > > > (both places).
> > >
> > > Stuff like proc_fd_link() uses get_files_struct(), which grabs a
> > > reference to your current files_struct protected only by task_lock();
> > > and it doesn't use anything like READ_ONCE(), so even if the object
> > > lifetime is not a problem, get_files_struct() could potentially crash
> > > due to a double-read (reading task->files twice and assuming that the
> > > result will be the same). As far as I can tell, this procfs code also
> > > works on kernel threads.
> > >
> > > > > > + if (!mmget_not_zero(ctx->sqo_mm)) {
> > > > > > + ret = -EFAULT;
> > > > > > + goto err;
> > > > > > + }
> > > > > > +
> > > > > > + use_mm(ctx->sqo_mm);
> > > > > > + set_fs(USER_DS);
> > > > > > +
> > > > > > + ret = __io_submit_sqe(ctx, req, s, false);
> > > > > > +
> > > > > > + set_fs(old_fs);
> > > > > > + unuse_mm(ctx->sqo_mm);
> > > > > > + mmput(ctx->sqo_mm);
> > > > > > +err:
> > > > > > + if (ret) {
> > > > > > + io_cqring_add_event(ctx, user_data, ret, 0);
> > > > > > + io_free_req(req);
> > > > > > + }
> > > > > > + current->files = old_files;
> > > > > > +}
> > > > >
> > > > > [...]
> > > > > > +static int io_sq_offload_start(struct io_ring_ctx *ctx)
> > > > > > +{
> > > > > > + int ret;
> > > > > > +
> > > > > > + ctx->sqo_mm = current->mm;
> > > > >
> > > > > What keeps this thing alive?
> > > >
> > > > I think we're deadling with the same thing as the files below, I'll
> > > > defer to that.
> > > >
> > > > > > + /*
> > > > > > + * This is safe since 'current' has the fd installed, and if that gets
> > > > > > + * closed on exit, then fops->release() is invoked which waits for the
> > > > > > + * async contexts to flush and exit before exiting.
> > > > > > + */
> > > > > > + ret = -EBADF;
> > > > > > + ctx->sqo_files = current->files;
> > > > > > + if (!ctx->sqo_files)
> > > > > > + goto err;
> > > > >
> > > > > That's gnarly. Adding Al Viro to the thread.
> > > > >
> > > > > I think you misunderstand the semantics of f_op->release. The ->flush
> > > > > handler is invoked whenever a file descriptor is closed through
> > > > > filp_close() (via deletion of the files_struct, sys_close(),
> > > > > sys_dup2(), ...), so if you had used that one, _maybe_ this would
> > > > > work. But the ->release handler only runs when the _last_ reference to
> > > > > a struct file has been dropped - so you can, for example, fork() a
> > > > > child, then exit() in the parent, and the ->release handler isn't
> > > > > invoked. So I don't see how this can work.
> > > >
> > > > The anonfd is CLOEXEC. The idea is exactly that it only runs when the
> > > > last reference to the file has been dropped. Not sure why you think I
> > > > need ->flush() here?
> > >
> > > Can't I just use fcntl(fd, F_SETFD, fd, 0) to clear the CLOEXEC flag?
> > > Or send the fd via SCM_RIGHTS?
> > >
> > > > > But even if you had abused ->flush for this instead: close_files()
> > > > > currently has a comment in it that claims that "this is the last
> > > > > reference to the files structure"; this change would make that claim
> > > > > untrue.
> > > >
> > > > Let me see if I can explain my intent better than that comment... We
> > > > know the parent who set up the io_uring instance will be around for as
> > > > long as io_uring instance persists.
> > >
> > > That's the part that I think is wrong: As far as I can tell, the
> > > parent can go away and you won't notice.
> > >
> > > Also, note that "the parent" is different things for ->files and ->mm.
> > > You can have a multithreaded process whose threads don't have the same
> > > ->files, or multiple process that share ->files without sharing ->mm,
> > > ...
> >
> > This had actually been get_files_struct() in early versions, and I had
> > reported to Jens that it allows something like
> >
> > int main() {
> > struct io_uring_params uring_params = {
> > .flags = IORING_SETUP_SQPOLL,
> > };
> > int uring_fd = syscall(425 /* io_uring_setup */, 16, &uring_params);
> > }
> >
> > to leak both the files_struct and the kthread, as the files_struct and
> > the uring context form a circular reference. I haven't really come up
> > with a good way to reconcile the requirements here; perhaps we need an
> > exit_uring() akin to exit_aio()?
>
> Oh, yuck. Uuuh... can we make "struct files_struct" doubly-refcounted,
> like "struct mm_struct"? One reference type to keep the contents
> intact (the reference type you normally use, and the type used by
> uring when the thread is running), and one reference type to just keep
> the struct itself existing, but without preserving its contents
> (reference held consistently by the uring thread)?
Something like this (completely untested); and then instead of the
current get_files_struct(), you'd do get_files_struct_weak(), and
while the thread is running, it protects the files_struct from dying
with tryget_weak_files_struct() / put_files_struct().
Al, do you have opinions on this?
===============
diff --git a/fs/file.c b/fs/file.c
index 3209ee271c41..fbf02ef2753d 100644
--- a/fs/file.c
+++ b/fs/file.c
@@ -281,6 +281,7 @@ struct files_struct *dup_fd(struct files_struct
*oldf, int *errorp)
if (!newf)
goto out;
+ kref_init(&newf->weak_refs);
atomic_set(&newf->count, 1);
spin_lock_init(&newf->file_lock);
@@ -410,6 +411,26 @@ struct files_struct *get_files_struct(struct
task_struct *task)
return files;
}
+static void free_files_struct(struct kref *ref) {
+ struct files_struct *files =
+ container_of(ref, struct files_struct, weak_refs);
+ kmem_cache_free(files_cachep, files);
+}
+
+void put_files_struct_weak(struct files_struct *files) {
+ kref_put(&files->weak_refs, free_files_struct);
+}
+
+struct files_struct *get_files_struct_weak(struct task_struct *task)
+{
+ struct files_struct *files = get_files_struct(task);
+ if (files) {
+ kref_get(&files->weak_refs);
+ put_files_struct(files);
+ }
+ return files;
+}
+
void put_files_struct(struct files_struct *files)
{
if (atomic_dec_and_test(&files->count)) {
@@ -418,10 +439,17 @@ void put_files_struct(struct files_struct *files)
/* free the arrays if they are not embedded */
if (fdt != &files->fdtab)
__free_fdtable(fdt);
- kmem_cache_free(files_cachep, files);
+ put_files_struct_weak(files);
}
}
+struct files_struct *tryget_weak_files_struct(struct files_struct *fs) {
+ if (atomic_inc_not_zero(&fs->count)) {
+ return fs;
+ }
+ return NULL;
+}
+
void reset_files_struct(struct files_struct *files)
{
struct task_struct *tsk = current;
@@ -448,6 +476,7 @@ void exit_files(struct task_struct *tsk)
struct files_struct init_files = {
.count = ATOMIC_INIT(1),
+ .weak_refs = KREF_INIT(1),
.fdt = &init_files.fdtab,
.fdtab = {
.max_fds = NR_OPEN_DEFAULT,
diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h
index f07c55ea0c22..6ad95a95cc0b 100644
--- a/include/linux/fdtable.h
+++ b/include/linux/fdtable.h
@@ -14,6 +14,7 @@
#include <linux/types.h>
#include <linux/init.h>
#include <linux/fs.h>
+#include <linux/kref.h>
#include <linux/atomic.h>
@@ -50,6 +51,7 @@ struct files_struct {
* read mostly part
*/
atomic_t count;
+ struct kref weak_refs;
bool resize_in_progress;
wait_queue_head_t resize_wait;
@@ -107,6 +109,9 @@ struct task_struct;
struct files_struct *get_files_struct(struct task_struct *);
void put_files_struct(struct files_struct *fs);
+void put_files_struct_weak(struct files_struct *files);
+struct files_struct *get_files_struct_weak(struct task_struct *);
+struct files_struct *tryget_weak_files_struct(struct files_struct *);
void reset_files_struct(struct files_struct *);
int unshare_files(struct files_struct **);
struct files_struct *dup_fd(struct files_struct *, int *) __latent_entropy;
===============
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply related
* Re: [PATCH 05/18] Add io_uring IO interface
From: Al Viro @ 2019-02-01 18:05 UTC (permalink / raw)
To: Jann Horn
Cc: Matt Mullins, axboe@kernel.dk, linux-fsdevel, linux-aio@kvack.org,
linux-block@vger.kernel.org, jmoyer@redhat.com,
linux-api@vger.kernel.org, hch@lst.de, linux-man@vger.kernel.org,
avi@scylladb.com
In-Reply-To: <CAG48ez0sJys=M1MR3DBd1DJ21-qHd2W7sB8+EvjhScWbtQnizA@mail.gmail.com>
On Fri, Feb 01, 2019 at 06:23:27PM +0100, Jann Horn wrote:
> > Oh, yuck. Uuuh... can we make "struct files_struct" doubly-refcounted,
> > like "struct mm_struct"? One reference type to keep the contents
> > intact (the reference type you normally use, and the type used by
> > uring when the thread is running), and one reference type to just keep
> > the struct itself existing, but without preserving its contents
> > (reference held consistently by the uring thread)?
>
> Something like this (completely untested); and then instead of the
> current get_files_struct(), you'd do get_files_struct_weak(), and
> while the thread is running, it protects the files_struct from dying
> with tryget_weak_files_struct() / put_files_struct().
>
> Al, do you have opinions on this?
Yes, but they are not fit for polite company. IMO the entire approach
is FUBAR; I'll post more detailed review, but what I'd seen so far is
a veto fodder.
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: [PATCH 05/18] Add io_uring IO interface
From: Florian Weimer @ 2019-02-01 18:20 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-aio, linux-block, linux-api, hch, jmoyer, avi, jannh
In-Reply-To: <20190201152414.23296-6-axboe@kernel.dk>
* Jens Axboe:
> +/*
> + * Filled with the offset for mmap(2)
> + */
> +struct io_sqring_offsets {
> + __u32 head;
> + __u32 tail;
> + __u32 ring_mask;
> + __u32 ring_entries;
> + __u32 flags;
> + __u32 dropped;
> + __u32 array;
> + __u32 resv[3];
> +};
> +
> +struct io_cqring_offsets {
> + __u32 head;
> + __u32 tail;
> + __u32 ring_mask;
> + __u32 ring_entries;
> + __u32 overflow;
> + __u32 cqes;
> + __u32 resv[4];
> +};
> +
> +/*
> + * io_uring_enter(2) flags
> + */
> +#define IORING_ENTER_GETEVENTS (1U << 0)
> +
> +/*
> + * Passed in for io_uring_setup(2). Copied back with updated info on success
> + */
> +struct io_uring_params {
> + __u32 sq_entries;
> + __u32 cq_entries;
> + __u32 flags;
> + __u32 resv[7];
> + struct io_sqring_offsets sq_off;
> + struct io_cqring_offsets cq_off;
> +};
I still think it might be a good idea to have one reserved __u64 field
for alignment purposes.
Thanks,
Florian
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: [PATCHSET v11] io_uring IO interface
From: Bart Van Assche @ 2019-02-01 22:52 UTC (permalink / raw)
To: Jens Axboe, linux-aio, linux-block, linux-api; +Cc: hch, jmoyer, avi, jannh
In-Reply-To: <20190201152414.23296-1-axboe@kernel.dk>
On Fri, 2019-02-01 at 08:23 -0700, Jens Axboe wrote:
> Here's v11 of the io_uring project. Main fixes in this release is a
> rework of how we grab the ctx->uring_lock, never using trylock for it in
> a user visible way. Outside of that, fixes around locking for the polled
> list when we hit -EAGAIN conditions on IO submit. This fixes list
> corruption issues with polling that some users have reported.
>
> As far as I'm concerned, this project is ready to get staged for 5.1.
> Please do review carefully so we can fix any minor nits that might still
> exist.
>
> The liburing git repo has a full set of man pages for this, though they
> could probably still use a bit of polish. I'd also like to see a
> io_uring(7) man page to describe the overall design of the project,
> expect that in the not-so-distant future. You can clone that here:
>
> git://git.kernel.dk/liburing
>
> Patches are against 5.0-rc4, and can also be found in my io_uring branch
> here:
>
> git://git.kernel.dk/linux-block io_uring
>
> Changes since v10:
> - Rework uring_lock locking
> - Ensure that async contexts lock when fiddling with polled lists
> - Minor tweak to io_iopoll_check() continue looping condition
> - Fold __io_uring_enter() into io_uring_enter()
Hi Jens,
Which goal(s) do you want to realize with this new API? Is the goal perhaps
to reduce the overhead of submitting I/O and receiving I/O completions?
If so, where can I find a comparison of the performance of this API with
libaio and synchronous I/O?
Thanks,
Bart.
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: [PATCHSET v11] io_uring IO interface
From: Jens Axboe @ 2019-02-02 13:52 UTC (permalink / raw)
To: Bart Van Assche, linux-aio, linux-block, linux-api
Cc: hch, jmoyer, avi, jannh
In-Reply-To: <1549061551.164323.17.camel@acm.org>
On 2/1/19 3:52 PM, Bart Van Assche wrote:
> On Fri, 2019-02-01 at 08:23 -0700, Jens Axboe wrote:
>> Here's v11 of the io_uring project. Main fixes in this release is a
>> rework of how we grab the ctx->uring_lock, never using trylock for it in
>> a user visible way. Outside of that, fixes around locking for the polled
>> list when we hit -EAGAIN conditions on IO submit. This fixes list
>> corruption issues with polling that some users have reported.
>>
>> As far as I'm concerned, this project is ready to get staged for 5.1.
>> Please do review carefully so we can fix any minor nits that might still
>> exist.
>>
>> The liburing git repo has a full set of man pages for this, though they
>> could probably still use a bit of polish. I'd also like to see a
>> io_uring(7) man page to describe the overall design of the project,
>> expect that in the not-so-distant future. You can clone that here:
>>
>> git://git.kernel.dk/liburing
>>
>> Patches are against 5.0-rc4, and can also be found in my io_uring branch
>> here:
>>
>> git://git.kernel.dk/linux-block io_uring
>>
>> Changes since v10:
>> - Rework uring_lock locking
>> - Ensure that async contexts lock when fiddling with polled lists
>> - Minor tweak to io_iopoll_check() continue looping condition
>> - Fold __io_uring_enter() into io_uring_enter()
>
> Hi Jens,
>
> Which goal(s) do you want to realize with this new API? Is the goal
> perhaps to reduce the overhead of submitting I/O and receiving I/O
> completions? If so, where can I find a comparison of the performance
> of this API with libaio and synchronous I/O?
Yes, it's to reduce the overhead of the interface, since aio isn't very
good. It's also to provide something that's more feature rich. It's
embarassing that we STILL don't have async buffered IO support, for
instance. With this infrastructure, we do.
In terms of benchmarks against aio, I included some in the v5 posting:
https://lore.kernel.org/linux-block/20190116175003.17880-1-axboe@kernel.dk/
I haven't done any against sync IO, but since you can use this interface
in a sync manner (just have min_complete == to_submit), there won't be
much of a difference, if any. If you use some of the more advanced
features, like pre-mapped buffers, it'll be faster than the current sync
IO.
--
Jens Axboe
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: [PATCH 13/18] io_uring: add file set registration
From: Al Viro @ 2019-02-04 2:56 UTC (permalink / raw)
To: Jann Horn
Cc: Jens Axboe, linux-aio, linux-block, Linux API, hch, jmoyer, avi,
linux-fsdevel
In-Reply-To: <CAG48ez2kLWUEambWesQQaeVn_u=PtjuMeDUc49yamhYspfd7Sg@mail.gmail.com>
On Wed, Jan 30, 2019 at 02:29:05AM +0100, Jann Horn wrote:
> On Tue, Jan 29, 2019 at 8:27 PM Jens Axboe <axboe@kernel.dk> wrote:
> > We normally have to fget/fput for each IO we do on a file. Even with
> > the batching we do, the cost of the atomic inc/dec of the file usage
> > count adds up.
> >
> > This adds IORING_REGISTER_FILES, and IORING_UNREGISTER_FILES opcodes
> > for the io_uring_register(2) system call. The arguments passed in must
> > be an array of __s32 holding file descriptors, and nr_args should hold
> > the number of file descriptors the application wishes to pin for the
> > duration of the io_uring context (or until IORING_UNREGISTER_FILES is
> > called).
> >
> > When used, the application must set IOSQE_FIXED_FILE in the sqe->flags
> > member. Then, instead of setting sqe->fd to the real fd, it sets sqe->fd
> > to the index in the array passed in to IORING_REGISTER_FILES.
> >
> > Files are automatically unregistered when the io_uring context is
> > torn down. An application need only unregister if it wishes to
> > register a new set of fds.
>
> Crazy idea:
>
> Taking a step back, at a high level, basically this patch creates sort
> of the same difference that you get when you compare the following
> scenarios for normal multithreaded I/O in userspace:
> This kinda makes me wonder whether this is really something that
> should be implemented specifically for the io_uring API, or whether it
> would make sense to somehow handle part of this in the generic VFS
> code and give the user the ability to prepare a new files_struct that
> can then be transferred to the worker thread, or something like
> that... I'm not sure whether there's a particularly clean way to do
> that though.
Using files_struct for that opens a can of worms you really don't
want to touch.
Consider the following scenario with any variant of this interface:
* create io_uring fd.
* send an SCM_RIGHTS with that fd to AF_UNIX socket.
* add the descriptor of that AF_UNIX socket to your fd
* close AF_UNIX fd, close io_uring fd.
Voila - you've got a shiny leak. No ->release() is called for
anyone (and you really don't want to do that on ->flush(), because
otherwise a library helper doing e.g. system("/bin/date") will tear
down all the io_uring in your process). The socket is held by
the reference you've stashed into io_uring (whichever way you do
that). io_uring is held by the reference you've stashed into
SCM_RIGHTS datagram in queue of the socket.
No matter what, you need net/unix/garbage.c to be aware of that stuff.
And getting files_struct lifetime mixed into that would be beyond
any reason.
The only reason for doing that as a descriptor table would be
avoiding the cost of fget() in whatever uses it, right? Since
those are *not* the normal syscalls (and fdget() really should not
be used anywhere other than the very top of syscall's call chain -
that's another reason why tossing file_struct around like that
is insane) and since the benefit is all due to the fact that it's
*NOT* shared, *NOT* modified in parallel, etc., allowing us to
treat file references as stable... why the hell use the descriptor
tables at all?
All you need is an array of struct file *, explicitly populated.
With net/unix/garbage.c aware of such beasts. Guess what? We
do have such an object already. The one net/unix/garbage.c is
working with. SCM_RIGHTS datagrams, that is.
IOW, can't we give those io_uring descriptors associated struct
unix_sock? No socket descriptors, no struct socket (probably),
just the AF_UNIX-specific part thereof. Then teach
unix_inflight()/unix_notinflight() about getting unix_sock out
of these guys (incidentally, both would seem to benefit from
_not_ touching unix_gc_lock in case when there's no unix_sock
attached to file we are dealing with - I might be missing
something very subtle about barriers there, but it doesn't
look likely).
And make that (i.e. registering the descriptors) mandatory.
Hell, combine that with creating io_uring fd, if we really
care about the syscall count. Benefits:
* no file_struct refcount wanking
* no fget()/fput() (conditional, at that) from kernel
threads
* no CLOEXEC-dependent anything; just the teardown
on the final fput(), whichever way it comes.
* no fun with duelling garbage collectors.
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* [RFC PATCH 1/4] glibc: Perform rseq(2) registration at C startup and thread creation (v7)
From: Mathieu Desnoyers @ 2019-02-04 21:28 UTC (permalink / raw)
To: Carlos O'Donell
Cc: Florian Weimer, Joseph Myers, Szabolcs Nagy, libc-alpha,
Mathieu Desnoyers, Thomas Gleixner, Ben Maurer, Peter Zijlstra,
Paul E. McKenney, Boqun Feng, Will Deacon, Dave Watson,
Paul Turner, Rich Felker, linux-kernel, linux-api
Register rseq(2) TLS for each thread (including main), and unregister
for each thread (excluding main). "rseq" stands for Restartable
Sequences.
See the rseq(2) man page proposed here:
https://lkml.org/lkml/2018/9/19/647
This patch is based on glibc-2.29. The rseq(2) system call was merged
into Linux 4.18.
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
CC: Carlos O'Donell <carlos@redhat.com>
CC: Florian Weimer <fweimer@redhat.com>
CC: Joseph Myers <joseph@codesourcery.com>
CC: Szabolcs Nagy <szabolcs.nagy@arm.com>
CC: Thomas Gleixner <tglx@linutronix.de>
CC: Ben Maurer <bmaurer@fb.com>
CC: Peter Zijlstra <peterz@infradead.org>
CC: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
CC: Boqun Feng <boqun.feng@gmail.com>
CC: Will Deacon <will.deacon@arm.com>
CC: Dave Watson <davejwatson@fb.com>
CC: Paul Turner <pjt@google.com>
CC: Rich Felker <dalias@libc.org>
CC: libc-alpha@sourceware.org
CC: linux-kernel@vger.kernel.org
CC: linux-api@vger.kernel.org
---
Changes since v1:
- Move __rseq_refcount to an extra field at the end of __rseq_abi to
eliminate one symbol.
All libraries/programs which try to register rseq (glibc,
early-adopter applications, early-adopter libraries) should use the
rseq refcount. It becomes part of the ABI within a user-space
process, but it's not part of the ABI shared with the kernel per se.
- Restructure how this code is organized so glibc keeps building on
non-Linux targets.
- Use non-weak symbol for __rseq_abi.
- Move rseq registration/unregistration implementation into its own
nptl/rseq.c compile unit.
- Move __rseq_abi symbol under GLIBC_2.29.
Changes since v2:
- Move __rseq_refcount to its own symbol, which is less ugly than
trying to play tricks with the rseq uapi.
- Move __rseq_abi from nptl to csu (C start up), so it can be used
across glibc, including memory allocator and sched_getcpu(). The
__rseq_refcount symbol is kept in nptl, because there is no reason
to use it elsewhere in glibc.
Changes since v3:
- Set __rseq_refcount TLS to 1 on register/set to 0 on unregister
because glibc is the first/last user.
- Unconditionally register/unregister rseq at thread start/exit, because
glibc is the first/last user.
- Add missing abilist items.
- Rebase on glibc master commit a502c5294.
- Add NEWS entry.
Changes since v4:
- Do not use "weak" symbols for __rseq_abi and __rseq_refcount. Based on
"System V Application Binary Interface", weak only affects the link
editor, not the dynamic linker.
- Install a new sys/rseq.h system header on Linux, which contains the
RSEQ_SIG definition, __rseq_abi declaration and __rseq_refcount
declaration. Move those definition/declarations from rseq-internal.h
to the installed sys/rseq.h header.
- Considering that rseq is only available on Linux, move csu/rseq.c to
sysdeps/unix/sysv/linux/rseq-sym.c.
- Move __rseq_refcount from nptl/rseq.c to
sysdeps/unix/sysv/linux/rseq-sym.c, so it is only defined on Linux.
- Move both ABI definitions for __rseq_abi and __rseq_refcount to
sysdeps/unix/sysv/linux/Versions, so they only appear on Linux.
- Document __rseq_abi and __rseq_refcount volatile.
- Document the RSEQ_SIG signature define.
- Move registration functions from rseq.c to rseq-internal.h static
inline functions. Introduce empty stubs in misc/rseq-internal.h,
which can be overridden by architecture code in
sysdeps/unix/sysv/linux/rseq-internal.h.
- Rename __rseq_register_current_thread and __rseq_unregister_current_thread
to rseq_register_current_thread and rseq_unregister_current_thread,
now that those are only visible as internal static inline functions.
- Invoke rseq_register_current_thread() from libc-start.c LIBC_START_MAIN
rather than nptl init, so applications not linked against
libpthread.so have rseq registered for their main() thread. Note that
it is invoked separately for SHARED and !SHARED builds.
Changes since v5:
- Replace __rseq_refcount by __rseq_lib_abi, which contains two
uint32_t: register_state and refcount. The "register_state" field
allows inhibiting rseq registration from signal handlers nested on top
of glibc registration and occuring after rseq unregistration by glibc.
- Introduce enum rseq_register_state, which contains the states allowed
for the struct rseq_lib_abi register_state field.
Changes since v6:
- Introduce bits/rseq.h to define RSEQ_SIG for each architecture.
The generic bits/rseq.h does not define RSEQ_SIG, meaning that each
architecture implementing rseq needs to implement bits/rseq.h.
- Rename enum item RSEQ_REGISTER_NESTED to RSEQ_REGISTER_ONGOING.
- Port to glibc-2.29.
---
NEWS | 11 +++
csu/libc-start.c | 12 ++-
misc/Makefile | 3 +-
misc/rseq-internal.h | 34 +++++++
nptl/pthread_create.c | 9 ++
sysdeps/unix/sysv/linux/Makefile | 4 +-
sysdeps/unix/sysv/linux/Versions | 4 +
sysdeps/unix/sysv/linux/aarch64/bits/rseq.h | 24 +++++
sysdeps/unix/sysv/linux/aarch64/libc.abilist | 2 +
sysdeps/unix/sysv/linux/alpha/libc.abilist | 2 +
sysdeps/unix/sysv/linux/arm/bits/rseq.h | 24 +++++
sysdeps/unix/sysv/linux/arm/libc.abilist | 2 +
sysdeps/unix/sysv/linux/bits/rseq.h | 24 +++++
sysdeps/unix/sysv/linux/hppa/libc.abilist | 2 +
sysdeps/unix/sysv/linux/i386/libc.abilist | 2 +
sysdeps/unix/sysv/linux/ia64/libc.abilist | 2 +
.../sysv/linux/m68k/coldfire/libc.abilist | 2 +
.../unix/sysv/linux/m68k/m680x0/libc.abilist | 2 +
.../unix/sysv/linux/microblaze/libc.abilist | 2 +
sysdeps/unix/sysv/linux/mips/bits/rseq.h | 24 +++++
.../sysv/linux/mips/mips32/fpu/libc.abilist | 2 +
.../sysv/linux/mips/mips32/nofpu/libc.abilist | 2 +
.../sysv/linux/mips/mips64/n32/libc.abilist | 2 +
.../sysv/linux/mips/mips64/n64/libc.abilist | 2 +
sysdeps/unix/sysv/linux/nios2/libc.abilist | 2 +
sysdeps/unix/sysv/linux/powerpc/bits/rseq.h | 24 +++++
.../linux/powerpc/powerpc32/fpu/libc.abilist | 2 +
.../powerpc/powerpc32/nofpu/libc.abilist | 2 +
.../linux/powerpc/powerpc64/be/libc.abilist | 2 +
.../linux/powerpc/powerpc64/le/libc.abilist | 2 +
.../unix/sysv/linux/riscv/rv64/libc.abilist | 2 +
sysdeps/unix/sysv/linux/rseq-internal.h | 91 +++++++++++++++++++
sysdeps/unix/sysv/linux/rseq-sym.c | 54 +++++++++++
sysdeps/unix/sysv/linux/s390/bits/rseq.h | 24 +++++
.../unix/sysv/linux/s390/s390-32/libc.abilist | 2 +
.../unix/sysv/linux/s390/s390-64/libc.abilist | 2 +
sysdeps/unix/sysv/linux/sh/libc.abilist | 2 +
.../sysv/linux/sparc/sparc32/libc.abilist | 2 +
.../sysv/linux/sparc/sparc64/libc.abilist | 2 +
sysdeps/unix/sysv/linux/sys/rseq.h | 65 +++++++++++++
sysdeps/unix/sysv/linux/x86/bits/rseq.h | 24 +++++
.../unix/sysv/linux/x86_64/64/libc.abilist | 2 +
.../unix/sysv/linux/x86_64/x32/libc.abilist | 2 +
43 files changed, 501 insertions(+), 6 deletions(-)
create mode 100644 misc/rseq-internal.h
create mode 100644 sysdeps/unix/sysv/linux/aarch64/bits/rseq.h
create mode 100644 sysdeps/unix/sysv/linux/arm/bits/rseq.h
create mode 100644 sysdeps/unix/sysv/linux/bits/rseq.h
create mode 100644 sysdeps/unix/sysv/linux/mips/bits/rseq.h
create mode 100644 sysdeps/unix/sysv/linux/powerpc/bits/rseq.h
create mode 100644 sysdeps/unix/sysv/linux/rseq-internal.h
create mode 100644 sysdeps/unix/sysv/linux/rseq-sym.c
create mode 100644 sysdeps/unix/sysv/linux/s390/bits/rseq.h
create mode 100644 sysdeps/unix/sysv/linux/sys/rseq.h
create mode 100644 sysdeps/unix/sysv/linux/x86/bits/rseq.h
diff --git a/NEWS b/NEWS
index 912a9bdc0f..0608c60f7d 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,17 @@ See the end for copying conditions.
Please send GNU C library bug reports via <https://sourceware.org/bugzilla/>
using `glibc' in the "product" field.
\f
+Version 2.30
+
+Major new features:
+
+* Support for automatically registering threads with the Linux rseq(2)
+ system call has been added. This system call is implemented starting
+ from Linux 4.18. In order to be activated, it requires that glibc is built
+ against kernel headers that include this system call, and that glibc
+ detects availability of that system call at runtime.
+
+\f
Version 2.29
Major new features:
diff --git a/csu/libc-start.c b/csu/libc-start.c
index 5d9c3675fa..8680afc0ef 100644
--- a/csu/libc-start.c
+++ b/csu/libc-start.c
@@ -22,6 +22,7 @@
#include <ldsodefs.h>
#include <exit-thread.h>
#include <libc-internal.h>
+#include <rseq-internal.h>
#include <elf/dl-tunables.h>
@@ -140,7 +141,10 @@ LIBC_START_MAIN (int (*main) (int, char **, char ** MAIN_AUXVEC_DECL),
__libc_multiple_libcs = &_dl_starting_up && !_dl_starting_up;
-#ifndef SHARED
+#ifdef SHARED
+ /* Register rseq ABI to the kernel. */
+ (void) rseq_register_current_thread ();
+#else
_dl_relocate_static_pie ();
char **ev = &argv[argc + 1];
@@ -218,6 +222,9 @@ LIBC_START_MAIN (int (*main) (int, char **, char ** MAIN_AUXVEC_DECL),
}
# endif
+ /* Register rseq ABI to the kernel. */
+ (void) rseq_register_current_thread ();
+
/* Initialize libpthread if linked in. */
if (__pthread_initialize_minimal != NULL)
__pthread_initialize_minimal ();
@@ -230,8 +237,7 @@ LIBC_START_MAIN (int (*main) (int, char **, char ** MAIN_AUXVEC_DECL),
# else
__pointer_chk_guard_local = pointer_chk_guard;
# endif
-
-#endif /* !SHARED */
+#endif
/* Register the destructor of the dynamic linker if there is any. */
if (__glibc_likely (rtld_fini != NULL))
diff --git a/misc/Makefile b/misc/Makefile
index cf0daa1161..0ae1dbaf80 100644
--- a/misc/Makefile
+++ b/misc/Makefile
@@ -36,7 +36,8 @@ headers := sys/uio.h bits/uio-ext.h bits/uio_lim.h \
syslog.h sys/syslog.h \
bits/syslog.h bits/syslog-ldbl.h bits/syslog-path.h bits/error.h \
bits/select2.h bits/hwcap.h sys/auxv.h \
- sys/sysmacros.h bits/sysmacros.h bits/types/struct_iovec.h
+ sys/sysmacros.h bits/sysmacros.h bits/types/struct_iovec.h \
+ rseq-internal.h
routines := brk sbrk sstk ioctl \
readv writev preadv preadv64 pwritev pwritev64 \
diff --git a/misc/rseq-internal.h b/misc/rseq-internal.h
new file mode 100644
index 0000000000..915122e4bf
--- /dev/null
+++ b/misc/rseq-internal.h
@@ -0,0 +1,34 @@
+/* Copyright (C) 2018 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2018.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef RSEQ_INTERNAL_H
+#define RSEQ_INTERNAL_H
+
+static inline int
+rseq_register_current_thread (void)
+{
+ return -1;
+}
+
+static inline int
+rseq_unregister_current_thread (void)
+{
+ return -1;
+}
+
+#endif /* rseq-internal.h */
diff --git a/nptl/pthread_create.c b/nptl/pthread_create.c
index 2bd2b10727..90b3419390 100644
--- a/nptl/pthread_create.c
+++ b/nptl/pthread_create.c
@@ -33,6 +33,7 @@
#include <default-sched.h>
#include <futex-internal.h>
#include <tls-setup.h>
+#include <rseq-internal.h>
#include "libioP.h"
#include <shlib-compat.h>
@@ -378,6 +379,7 @@ __free_tcb (struct pthread *pd)
START_THREAD_DEFN
{
struct pthread *pd = START_THREAD_SELF;
+ bool has_rseq = false;
#if HP_TIMING_AVAIL
/* Remember the time when the thread was started. */
@@ -396,6 +398,9 @@ START_THREAD_DEFN
if (__glibc_unlikely (atomic_exchange_acq (&pd->setxid_futex, 0) == -2))
futex_wake (&pd->setxid_futex, 1, FUTEX_PRIVATE);
+ /* Register rseq TLS to the kernel. */
+ has_rseq = !rseq_register_current_thread ();
+
#ifdef __NR_set_robust_list
# ifndef __ASSUME_SET_ROBUST_LIST
if (__set_robust_list_avail >= 0)
@@ -573,6 +578,10 @@ START_THREAD_DEFN
}
#endif
+ /* Unregister rseq TLS from kernel. */
+ if (has_rseq && rseq_unregister_current_thread ())
+ abort();
+
advise_stack_range (pd->stackblock, pd->stackblock_size, (uintptr_t) pd,
pd->guardsize);
diff --git a/sysdeps/unix/sysv/linux/Makefile b/sysdeps/unix/sysv/linux/Makefile
index 5f8c2c7c7d..5b541469ec 100644
--- a/sysdeps/unix/sysv/linux/Makefile
+++ b/sysdeps/unix/sysv/linux/Makefile
@@ -1,5 +1,5 @@
ifeq ($(subdir),csu)
-sysdep_routines += errno-loc
+sysdep_routines += errno-loc rseq-sym
endif
ifeq ($(subdir),assert)
@@ -48,7 +48,7 @@ sysdep_headers += sys/mount.h sys/acct.h sys/sysctl.h \
bits/termios-c_iflag.h bits/termios-c_oflag.h \
bits/termios-baud.h bits/termios-c_cflag.h \
bits/termios-c_lflag.h bits/termios-tcflow.h \
- bits/termios-misc.h
+ bits/termios-misc.h sys/rseq.h bits/rseq.h
tests += tst-clone tst-clone2 tst-clone3 tst-fanotify tst-personality \
tst-quota tst-sync_file_range tst-sysconf-iov_max tst-ttyname \
diff --git a/sysdeps/unix/sysv/linux/Versions b/sysdeps/unix/sysv/linux/Versions
index f1e12d9c69..ad88c2b7ff 100644
--- a/sysdeps/unix/sysv/linux/Versions
+++ b/sysdeps/unix/sysv/linux/Versions
@@ -174,6 +174,10 @@ libc {
GLIBC_2.29 {
getcpu;
}
+ GLIBC_2.30 {
+ __rseq_abi;
+ __rseq_lib_abi;
+ }
GLIBC_PRIVATE {
# functions used in other libraries
__syscall_rt_sigqueueinfo;
diff --git a/sysdeps/unix/sysv/linux/aarch64/bits/rseq.h b/sysdeps/unix/sysv/linux/aarch64/bits/rseq.h
new file mode 100644
index 0000000000..543bc5388a
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/aarch64/bits/rseq.h
@@ -0,0 +1,24 @@
+/* Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2019.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef _SYS_RSEQ_H
+# error "Never use <bits/rseq.h> directly; include <sys/rseq.h> instead."
+#endif
+
+/* Signature required before each abort handler code. */
+#define RSEQ_SIG 0xd428bc00 /* BRK #0x45E0. */
diff --git a/sysdeps/unix/sysv/linux/aarch64/libc.abilist b/sysdeps/unix/sysv/linux/aarch64/libc.abilist
index 9c330f325e..bc937f585d 100644
--- a/sysdeps/unix/sysv/linux/aarch64/libc.abilist
+++ b/sysdeps/unix/sysv/linux/aarch64/libc.abilist
@@ -2141,3 +2141,5 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
diff --git a/sysdeps/unix/sysv/linux/alpha/libc.abilist b/sysdeps/unix/sysv/linux/alpha/libc.abilist
index f630fa4c6f..89cc8b1cfb 100644
--- a/sysdeps/unix/sysv/linux/alpha/libc.abilist
+++ b/sysdeps/unix/sysv/linux/alpha/libc.abilist
@@ -2036,6 +2036,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/arm/bits/rseq.h b/sysdeps/unix/sysv/linux/arm/bits/rseq.h
new file mode 100644
index 0000000000..19d3755837
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/arm/bits/rseq.h
@@ -0,0 +1,24 @@
+/* Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2019.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef _SYS_RSEQ_H
+# error "Never use <bits/rseq.h> directly; include <sys/rseq.h> instead."
+#endif
+
+/* Signature required before each abort handler code. */
+#define RSEQ_SIG 0x53053053
diff --git a/sysdeps/unix/sysv/linux/arm/libc.abilist b/sysdeps/unix/sysv/linux/arm/libc.abilist
index b96f45590f..e5055f2d4e 100644
--- a/sysdeps/unix/sysv/linux/arm/libc.abilist
+++ b/sysdeps/unix/sysv/linux/arm/libc.abilist
@@ -126,6 +126,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.4 _Exit F
GLIBC_2.4 _IO_2_1_stderr_ D 0xa0
GLIBC_2.4 _IO_2_1_stdin_ D 0xa0
diff --git a/sysdeps/unix/sysv/linux/bits/rseq.h b/sysdeps/unix/sysv/linux/bits/rseq.h
new file mode 100644
index 0000000000..d60f02cfeb
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/bits/rseq.h
@@ -0,0 +1,24 @@
+/* Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2019.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef _SYS_RSEQ_H
+# error "Never use <bits/rseq.h> directly; include <sys/rseq.h> instead."
+#endif
+
+/* Each architecture supporting rseq should define RSEQ_SIG as a 32-bit
+ signature inserted before each rseq abort label in the code section. */
diff --git a/sysdeps/unix/sysv/linux/hppa/libc.abilist b/sysdeps/unix/sysv/linux/hppa/libc.abilist
index 088a8ee369..546d073cdb 100644
--- a/sysdeps/unix/sysv/linux/hppa/libc.abilist
+++ b/sysdeps/unix/sysv/linux/hppa/libc.abilist
@@ -1883,6 +1883,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/i386/libc.abilist b/sysdeps/unix/sysv/linux/i386/libc.abilist
index f7ff2c57b9..ac1de6e4b3 100644
--- a/sysdeps/unix/sysv/linux/i386/libc.abilist
+++ b/sysdeps/unix/sysv/linux/i386/libc.abilist
@@ -2048,6 +2048,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/ia64/libc.abilist b/sysdeps/unix/sysv/linux/ia64/libc.abilist
index becd8b1033..cc3445b958 100644
--- a/sysdeps/unix/sysv/linux/ia64/libc.abilist
+++ b/sysdeps/unix/sysv/linux/ia64/libc.abilist
@@ -1917,6 +1917,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/m68k/coldfire/libc.abilist b/sysdeps/unix/sysv/linux/m68k/coldfire/libc.abilist
index 74e42a5209..f7e28bd5a0 100644
--- a/sysdeps/unix/sysv/linux/m68k/coldfire/libc.abilist
+++ b/sysdeps/unix/sysv/linux/m68k/coldfire/libc.abilist
@@ -127,6 +127,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.4 _Exit F
GLIBC_2.4 _IO_2_1_stderr_ D 0x98
GLIBC_2.4 _IO_2_1_stdin_ D 0x98
diff --git a/sysdeps/unix/sysv/linux/m68k/m680x0/libc.abilist b/sysdeps/unix/sysv/linux/m68k/m680x0/libc.abilist
index 4af5a74e8a..b8f00f6111 100644
--- a/sysdeps/unix/sysv/linux/m68k/m680x0/libc.abilist
+++ b/sysdeps/unix/sysv/linux/m68k/m680x0/libc.abilist
@@ -1992,6 +1992,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/microblaze/libc.abilist b/sysdeps/unix/sysv/linux/microblaze/libc.abilist
index ccef673fd2..19f191434f 100644
--- a/sysdeps/unix/sysv/linux/microblaze/libc.abilist
+++ b/sysdeps/unix/sysv/linux/microblaze/libc.abilist
@@ -2133,3 +2133,5 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
diff --git a/sysdeps/unix/sysv/linux/mips/bits/rseq.h b/sysdeps/unix/sysv/linux/mips/bits/rseq.h
new file mode 100644
index 0000000000..19d3755837
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/mips/bits/rseq.h
@@ -0,0 +1,24 @@
+/* Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2019.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef _SYS_RSEQ_H
+# error "Never use <bits/rseq.h> directly; include <sys/rseq.h> instead."
+#endif
+
+/* Signature required before each abort handler code. */
+#define RSEQ_SIG 0x53053053
diff --git a/sysdeps/unix/sysv/linux/mips/mips32/fpu/libc.abilist b/sysdeps/unix/sysv/linux/mips/mips32/fpu/libc.abilist
index 1054bb599e..fe43507f55 100644
--- a/sysdeps/unix/sysv/linux/mips/mips32/fpu/libc.abilist
+++ b/sysdeps/unix/sysv/linux/mips/mips32/fpu/libc.abilist
@@ -1970,6 +1970,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/mips/mips32/nofpu/libc.abilist b/sysdeps/unix/sysv/linux/mips/mips32/nofpu/libc.abilist
index 4f5b5ffebf..b247c6ea9b 100644
--- a/sysdeps/unix/sysv/linux/mips/mips32/nofpu/libc.abilist
+++ b/sysdeps/unix/sysv/linux/mips/mips32/nofpu/libc.abilist
@@ -1968,6 +1968,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/mips/mips64/n32/libc.abilist b/sysdeps/unix/sysv/linux/mips/mips64/n32/libc.abilist
index 943aee58d4..5339ca52b6 100644
--- a/sysdeps/unix/sysv/linux/mips/mips64/n32/libc.abilist
+++ b/sysdeps/unix/sysv/linux/mips/mips64/n32/libc.abilist
@@ -1976,6 +1976,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/mips/mips64/n64/libc.abilist b/sysdeps/unix/sysv/linux/mips/mips64/n64/libc.abilist
index 17a5d17ef9..11f24eb440 100644
--- a/sysdeps/unix/sysv/linux/mips/mips64/n64/libc.abilist
+++ b/sysdeps/unix/sysv/linux/mips/mips64/n64/libc.abilist
@@ -1971,6 +1971,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/nios2/libc.abilist b/sysdeps/unix/sysv/linux/nios2/libc.abilist
index 4d62a540fd..fd223bfc44 100644
--- a/sysdeps/unix/sysv/linux/nios2/libc.abilist
+++ b/sysdeps/unix/sysv/linux/nios2/libc.abilist
@@ -2174,3 +2174,5 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
diff --git a/sysdeps/unix/sysv/linux/powerpc/bits/rseq.h b/sysdeps/unix/sysv/linux/powerpc/bits/rseq.h
new file mode 100644
index 0000000000..19d3755837
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/powerpc/bits/rseq.h
@@ -0,0 +1,24 @@
+/* Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2019.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef _SYS_RSEQ_H
+# error "Never use <bits/rseq.h> directly; include <sys/rseq.h> instead."
+#endif
+
+/* Signature required before each abort handler code. */
+#define RSEQ_SIG 0x53053053
diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc32/fpu/libc.abilist b/sysdeps/unix/sysv/linux/powerpc/powerpc32/fpu/libc.abilist
index ecc2d6fa13..cc53178e81 100644
--- a/sysdeps/unix/sysv/linux/powerpc/powerpc32/fpu/libc.abilist
+++ b/sysdeps/unix/sysv/linux/powerpc/powerpc32/fpu/libc.abilist
@@ -1996,6 +1996,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc32/nofpu/libc.abilist b/sysdeps/unix/sysv/linux/powerpc/powerpc32/nofpu/libc.abilist
index f5830f9c33..2de3134bc7 100644
--- a/sysdeps/unix/sysv/linux/powerpc/powerpc32/nofpu/libc.abilist
+++ b/sysdeps/unix/sysv/linux/powerpc/powerpc32/nofpu/libc.abilist
@@ -2000,6 +2000,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc64/be/libc.abilist b/sysdeps/unix/sysv/linux/powerpc/powerpc64/be/libc.abilist
index 633d8f4792..aae3def700 100644
--- a/sysdeps/unix/sysv/linux/powerpc/powerpc64/be/libc.abilist
+++ b/sysdeps/unix/sysv/linux/powerpc/powerpc64/be/libc.abilist
@@ -126,6 +126,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 _Exit F
GLIBC_2.3 _IO_2_1_stderr_ D 0xe0
GLIBC_2.3 _IO_2_1_stdin_ D 0xe0
diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc64/le/libc.abilist b/sysdeps/unix/sysv/linux/powerpc/powerpc64/le/libc.abilist
index 2c712636ef..8d582a3a9b 100644
--- a/sysdeps/unix/sysv/linux/powerpc/powerpc64/le/libc.abilist
+++ b/sysdeps/unix/sysv/linux/powerpc/powerpc64/le/libc.abilist
@@ -2231,3 +2231,5 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
diff --git a/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist b/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist
index 195bc8b2cf..155953f6cf 100644
--- a/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist
+++ b/sysdeps/unix/sysv/linux/riscv/rv64/libc.abilist
@@ -2103,3 +2103,5 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
diff --git a/sysdeps/unix/sysv/linux/rseq-internal.h b/sysdeps/unix/sysv/linux/rseq-internal.h
new file mode 100644
index 0000000000..d676da3701
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/rseq-internal.h
@@ -0,0 +1,91 @@
+/* Copyright (C) 2018 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2018.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef RSEQ_INTERNAL_H
+#define RSEQ_INTERNAL_H
+
+#include <sysdep.h>
+
+#ifdef __NR_rseq
+
+#include <errno.h>
+#include <sys/rseq.h>
+
+static inline int
+rseq_register_current_thread (void)
+{
+ int rc, ret = 0;
+ INTERNAL_SYSCALL_DECL (err);
+
+ if (__rseq_abi.cpu_id == RSEQ_CPU_ID_REGISTRATION_FAILED)
+ return -1;
+ /* Temporarily prevent nested signal handlers from registering rseq. */
+ __rseq_lib_abi.register_state = RSEQ_REGISTER_ONGOING;
+ if (__rseq_lib_abi.refcount == UINT_MAX)
+ {
+ ret = -1;
+ goto end;
+ }
+ if (__rseq_lib_abi.refcount++)
+ goto end;
+ rc = INTERNAL_SYSCALL_CALL (rseq, err, &__rseq_abi, sizeof (struct rseq),
+ 0, RSEQ_SIG);
+ if (!rc)
+ goto end;
+ if (INTERNAL_SYSCALL_ERRNO (rc, err) != EBUSY)
+ __rseq_abi.cpu_id = RSEQ_CPU_ID_REGISTRATION_FAILED;
+ ret = -1;
+end:
+ __rseq_lib_abi.register_state = RSEQ_REGISTER_ALLOWED;
+ return ret;
+}
+
+static inline int
+rseq_unregister_current_thread (void)
+{
+ int rc, ret = 0;
+ INTERNAL_SYSCALL_DECL (err);
+
+ /* Setting __rseq_register_state = RSEQ_REGISTER_EXITING for the rest of the
+ thread lifetime. Ensures signal handlers nesting just before thread exit
+ don't try to register rseq. */
+ __rseq_lib_abi.register_state = RSEQ_REGISTER_EXITING;
+ __rseq_lib_abi.refcount = 0;
+ rc = INTERNAL_SYSCALL_CALL (rseq, err, &__rseq_abi, sizeof (struct rseq),
+ RSEQ_FLAG_UNREGISTER, RSEQ_SIG);
+ if (!rc)
+ goto end;
+ ret = -1;
+end:
+ return ret;
+}
+#else
+static inline int
+rseq_register_current_thread (void)
+{
+ return -1;
+}
+
+static inline int
+rseq_unregister_current_thread (void)
+{
+ return -1;
+}
+#endif
+
+#endif /* rseq-internal.h */
diff --git a/sysdeps/unix/sysv/linux/rseq-sym.c b/sysdeps/unix/sysv/linux/rseq-sym.c
new file mode 100644
index 0000000000..99b277e9d6
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/rseq-sym.c
@@ -0,0 +1,54 @@
+/* Copyright (C) 2018 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2018.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#include <sys/syscall.h>
+#include <stdint.h>
+
+#ifdef __NR_rseq
+#include <sys/rseq.h>
+#else
+
+enum rseq_cpu_id_state {
+ RSEQ_CPU_ID_UNINITIALIZED = -1,
+ RSEQ_CPU_ID_REGISTRATION_FAILED = -2,
+};
+
+/* linux/rseq.h defines struct rseq as aligned on 32 bytes. The kernel ABI
+ size is 20 bytes. */
+struct rseq {
+ uint32_t cpu_id_start;
+ uint32_t cpu_id;
+ uint64_t rseq_cs;
+ uint32_t flags;
+} __attribute__ ((aligned(4 * sizeof(uint64_t))));
+
+struct rseq_lib_abi
+{
+ uint32_t register_state;
+ uint32_t refcount;
+};
+
+#endif
+
+/* volatile because fields can be read/updated by the kernel. */
+__thread volatile struct rseq __rseq_abi = {
+ .cpu_id = RSEQ_CPU_ID_UNINITIALIZED,
+};
+
+/* volatile because fields can be read/updated by signal handlers. */
+__thread volatile struct rseq_lib_abi __rseq_lib_abi;
diff --git a/sysdeps/unix/sysv/linux/s390/bits/rseq.h b/sysdeps/unix/sysv/linux/s390/bits/rseq.h
new file mode 100644
index 0000000000..19d3755837
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/s390/bits/rseq.h
@@ -0,0 +1,24 @@
+/* Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2019.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef _SYS_RSEQ_H
+# error "Never use <bits/rseq.h> directly; include <sys/rseq.h> instead."
+#endif
+
+/* Signature required before each abort handler code. */
+#define RSEQ_SIG 0x53053053
diff --git a/sysdeps/unix/sysv/linux/s390/s390-32/libc.abilist b/sysdeps/unix/sysv/linux/s390/s390-32/libc.abilist
index 334def033c..42316d8666 100644
--- a/sysdeps/unix/sysv/linux/s390/s390-32/libc.abilist
+++ b/sysdeps/unix/sysv/linux/s390/s390-32/libc.abilist
@@ -2005,6 +2005,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/s390/s390-64/libc.abilist b/sysdeps/unix/sysv/linux/s390/s390-64/libc.abilist
index 536f4c4ced..c6c4e55a77 100644
--- a/sysdeps/unix/sysv/linux/s390/s390-64/libc.abilist
+++ b/sysdeps/unix/sysv/linux/s390/s390-64/libc.abilist
@@ -1911,6 +1911,8 @@ GLIBC_2.29 __fentry__ F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/sh/libc.abilist b/sysdeps/unix/sysv/linux/sh/libc.abilist
index 30ae3b6ebb..8652dfea59 100644
--- a/sysdeps/unix/sysv/linux/sh/libc.abilist
+++ b/sysdeps/unix/sysv/linux/sh/libc.abilist
@@ -1887,6 +1887,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/sparc/sparc32/libc.abilist b/sysdeps/unix/sysv/linux/sparc/sparc32/libc.abilist
index 68b107d080..95b58dfa67 100644
--- a/sysdeps/unix/sysv/linux/sparc/sparc32/libc.abilist
+++ b/sysdeps/unix/sysv/linux/sparc/sparc32/libc.abilist
@@ -1999,6 +1999,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/sparc/sparc64/libc.abilist b/sysdeps/unix/sysv/linux/sparc/sparc64/libc.abilist
index e5b6a4da50..bfd24f9d1c 100644
--- a/sysdeps/unix/sysv/linux/sparc/sparc64/libc.abilist
+++ b/sysdeps/unix/sysv/linux/sparc/sparc64/libc.abilist
@@ -1940,6 +1940,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/sys/rseq.h b/sysdeps/unix/sysv/linux/sys/rseq.h
new file mode 100644
index 0000000000..83c8976f50
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/sys/rseq.h
@@ -0,0 +1,65 @@
+/* Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2019.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef _SYS_RSEQ_H
+#define _SYS_RSEQ_H 1
+
+/* We use the structures declarations from the kernel headers. */
+#include <linux/rseq.h>
+/* Architecture-specific rseq signature. */
+#include <bits/rseq.h>
+#include <stdint.h>
+
+enum rseq_register_state
+{
+ /* Value RSEQ_REGISTER_ALLOWED means it is allowed to update
+ the refcount field and to register/unregister rseq. */
+ RSEQ_REGISTER_ALLOWED = 0,
+ /* Value RSEQ_REGISTER_ONGOING means a rseq registration is in progress,
+ so it is temporarily forbidden to update the refcount field or to
+ register/unregister rseq for this thread or signal handlers nested
+ on this thread. */
+ RSEQ_REGISTER_ONGOING = 1,
+ /* Value RSEQ_REGISTER_EXITING means it is forbidden to update the
+ refcount field or to register/unregister rseq for the rest of the
+ thread's lifetime. */
+ RSEQ_REGISTER_EXITING = 2,
+};
+
+struct rseq_lib_abi
+{
+ uint32_t register_state; /* enum rseq_register_state. */
+ /* The refcount field keeps track of rseq users, so early adopters
+ of rseq can cooperate amongst each other and with glibc to
+ share rseq thread registration. The refcount field can only be
+ updated when allowed by the value of field register_state.
+ Registering rseq should be performed when incrementing refcount
+ from 0 to 1, and unregistering rseq should be performed when
+ decrementing refcount from 1 to 0. */
+ uint32_t refcount;
+};
+
+/* volatile because fields can be read/updated by the kernel. */
+extern __thread volatile struct rseq __rseq_abi
+__attribute__ ((tls_model ("initial-exec")));
+
+/* volatile because fields can be read/updated by signal handlers. */
+extern __thread volatile struct rseq_lib_abi __rseq_lib_abi
+__attribute__ ((tls_model ("initial-exec")));
+
+#endif /* sys/rseq.h */
diff --git a/sysdeps/unix/sysv/linux/x86/bits/rseq.h b/sysdeps/unix/sysv/linux/x86/bits/rseq.h
new file mode 100644
index 0000000000..19d3755837
--- /dev/null
+++ b/sysdeps/unix/sysv/linux/x86/bits/rseq.h
@@ -0,0 +1,24 @@
+/* Copyright (C) 2019 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+ Contributed by Mathieu Desnoyers <mathieu.desnoyers@efficios.com>, 2019.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+
+#ifndef _SYS_RSEQ_H
+# error "Never use <bits/rseq.h> directly; include <sys/rseq.h> instead."
+#endif
+
+/* Signature required before each abort handler code. */
+#define RSEQ_SIG 0x53053053
diff --git a/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist b/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist
index 86dfb0c94d..e9f8411fb2 100644
--- a/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist
+++ b/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist
@@ -1898,6 +1898,8 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
GLIBC_2.3 __ctype_b_loc F
GLIBC_2.3 __ctype_tolower_loc F
GLIBC_2.3 __ctype_toupper_loc F
diff --git a/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist b/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist
index dd688263aa..f9432d07f1 100644
--- a/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist
+++ b/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist
@@ -2149,3 +2149,5 @@ GLIBC_2.28 thrd_yield F
GLIBC_2.29 getcpu F
GLIBC_2.29 posix_spawn_file_actions_addchdir_np F
GLIBC_2.29 posix_spawn_file_actions_addfchdir_np F
+GLIBC_2.30 __rseq_abi T 0x20
+GLIBC_2.30 __rseq_lib_abi T 0x8
--
2.17.1
^ permalink raw reply related
* [RFC PATCH 2/4] glibc: sched_getcpu(): use rseq cpu_id TLS on Linux
From: Mathieu Desnoyers @ 2019-02-04 21:29 UTC (permalink / raw)
To: Carlos O'Donell
Cc: Florian Weimer, Joseph Myers, Szabolcs Nagy, libc-alpha,
Mathieu Desnoyers, Thomas Gleixner, Ben Maurer, Peter Zijlstra,
Paul E. McKenney, Boqun Feng, Will Deacon, Dave Watson,
Paul Turner, linux-kernel, linux-api
In-Reply-To: <20190204212902.31474-1-mathieu.desnoyers@efficios.com>
When available, use the cpu_id field from __rseq_abi on Linux to
implement sched_getcpu(). Fall-back on the vgetcpu vDSO if unavailable.
Benchmarks:
x86-64: Intel E5-2630 v3@2.40GHz, 16-core, hyperthreading
glibc sched_getcpu(): 13.7 ns (baseline)
glibc sched_getcpu() using rseq: 2.5 ns (speedup: 5.5x)
inline load cpuid from __rseq_abi TLS: 0.8 ns (speedup: 17.1x)
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
CC: Carlos O'Donell <carlos@redhat.com>
CC: Florian Weimer <fweimer@redhat.com>
CC: Joseph Myers <joseph@codesourcery.com>
CC: Szabolcs Nagy <szabolcs.nagy@arm.com>
CC: Thomas Gleixner <tglx@linutronix.de>
CC: Ben Maurer <bmaurer@fb.com>
CC: Peter Zijlstra <peterz@infradead.org>
CC: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
CC: Boqun Feng <boqun.feng@gmail.com>
CC: Will Deacon <will.deacon@arm.com>
CC: Dave Watson <davejwatson@fb.com>
CC: Paul Turner <pjt@google.com>
CC: libc-alpha@sourceware.org
CC: linux-kernel@vger.kernel.org
CC: linux-api@vger.kernel.org
---
sysdeps/unix/sysv/linux/sched_getcpu.c | 25 +++++++++++++++++++++++--
1 file changed, 23 insertions(+), 2 deletions(-)
diff --git a/sysdeps/unix/sysv/linux/sched_getcpu.c b/sysdeps/unix/sysv/linux/sched_getcpu.c
index fb0d317f83..8bfb03778b 100644
--- a/sysdeps/unix/sysv/linux/sched_getcpu.c
+++ b/sysdeps/unix/sysv/linux/sched_getcpu.c
@@ -24,8 +24,8 @@
#endif
#include <sysdep-vdso.h>
-int
-sched_getcpu (void)
+static int
+vsyscall_sched_getcpu (void)
{
#ifdef __NR_getcpu
unsigned int cpu;
@@ -37,3 +37,24 @@ sched_getcpu (void)
return -1;
#endif
}
+
+#ifdef __NR_rseq
+#include <linux/rseq.h>
+
+extern __attribute__ ((tls_model ("initial-exec")))
+__thread volatile struct rseq __rseq_abi;
+
+int
+sched_getcpu (void)
+{
+ int cpu_id = __rseq_abi.cpu_id;
+
+ return cpu_id >= 0 ? cpu_id : vsyscall_sched_getcpu ();
+}
+#else
+int
+sched_getcpu (void)
+{
+ return vsyscall_sched_getcpu ();
+}
+#endif
--
2.17.1
^ permalink raw reply related
* Re: [PATCH 05/18] Add io_uring IO interface
From: Jeff Moyer @ 2019-02-04 23:22 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-aio, linux-block, linux-api, hch, avi, jannh
In-Reply-To: <20190201152414.23296-6-axboe@kernel.dk>
Hi, Jens,
Jens Axboe <axboe@kernel.dk> writes:
> +static void io_ring_ctx_free(struct io_ring_ctx *ctx)
> +{
> + destroy_workqueue(ctx->sqo_wq);
You need to make sure sqo_wq is non-null before calling
destroy_workqueue.
-Jeff
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: [PATCH 05/18] Add io_uring IO interface
From: Jeff Moyer @ 2019-02-04 23:52 UTC (permalink / raw)
To: Jens Axboe; +Cc: linux-aio, linux-block, linux-api, hch, avi, jannh
In-Reply-To: <x49sgx3t9yv.fsf@segfault.boston.devel.redhat.com>
Jeff Moyer <jmoyer@redhat.com> writes:
> Hi, Jens,
>
> Jens Axboe <axboe@kernel.dk> writes:
>
>> +static void io_ring_ctx_free(struct io_ring_ctx *ctx)
>> +{
>> + destroy_workqueue(ctx->sqo_wq);
>
> You need to make sure sqo_wq is non-null before calling
> destroy_workqueue.
You also need to check for non-null sqo_files and sqo_mm.
Basically, you're tearing things down twice if setup of the context
fails in some way.
-Jeff
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: [PATCH 13/18] io_uring: add file set registration
From: Jens Axboe @ 2019-02-05 2:19 UTC (permalink / raw)
To: Al Viro, Jann Horn
Cc: linux-aio, linux-block, Linux API, hch, jmoyer, avi,
linux-fsdevel
In-Reply-To: <20190204025612.GR2217@ZenIV.linux.org.uk>
On 2/3/19 7:56 PM, Al Viro wrote:
> On Wed, Jan 30, 2019 at 02:29:05AM +0100, Jann Horn wrote:
>> On Tue, Jan 29, 2019 at 8:27 PM Jens Axboe <axboe@kernel.dk> wrote:
>>> We normally have to fget/fput for each IO we do on a file. Even with
>>> the batching we do, the cost of the atomic inc/dec of the file usage
>>> count adds up.
>>>
>>> This adds IORING_REGISTER_FILES, and IORING_UNREGISTER_FILES opcodes
>>> for the io_uring_register(2) system call. The arguments passed in must
>>> be an array of __s32 holding file descriptors, and nr_args should hold
>>> the number of file descriptors the application wishes to pin for the
>>> duration of the io_uring context (or until IORING_UNREGISTER_FILES is
>>> called).
>>>
>>> When used, the application must set IOSQE_FIXED_FILE in the sqe->flags
>>> member. Then, instead of setting sqe->fd to the real fd, it sets sqe->fd
>>> to the index in the array passed in to IORING_REGISTER_FILES.
>>>
>>> Files are automatically unregistered when the io_uring context is
>>> torn down. An application need only unregister if it wishes to
>>> register a new set of fds.
>>
>> Crazy idea:
>>
>> Taking a step back, at a high level, basically this patch creates sort
>> of the same difference that you get when you compare the following
>> scenarios for normal multithreaded I/O in userspace:
>
>> This kinda makes me wonder whether this is really something that
>> should be implemented specifically for the io_uring API, or whether it
>> would make sense to somehow handle part of this in the generic VFS
>> code and give the user the ability to prepare a new files_struct that
>> can then be transferred to the worker thread, or something like
>> that... I'm not sure whether there's a particularly clean way to do
>> that though.
>
> Using files_struct for that opens a can of worms you really don't
> want to touch.
>
> Consider the following scenario with any variant of this interface:
> * create io_uring fd.
> * send an SCM_RIGHTS with that fd to AF_UNIX socket.
> * add the descriptor of that AF_UNIX socket to your fd
> * close AF_UNIX fd, close io_uring fd.
> Voila - you've got a shiny leak. No ->release() is called for
> anyone (and you really don't want to do that on ->flush(), because
> otherwise a library helper doing e.g. system("/bin/date") will tear
> down all the io_uring in your process). The socket is held by
> the reference you've stashed into io_uring (whichever way you do
> that). io_uring is held by the reference you've stashed into
> SCM_RIGHTS datagram in queue of the socket.
>
> No matter what, you need net/unix/garbage.c to be aware of that stuff.
> And getting files_struct lifetime mixed into that would be beyond
> any reason.
>
> The only reason for doing that as a descriptor table would be
> avoiding the cost of fget() in whatever uses it, right? Since
Right, the only purpose of this patch is to avoid doing fget/fput for
each IO.
> those are *not* the normal syscalls (and fdget() really should not
> be used anywhere other than the very top of syscall's call chain -
> that's another reason why tossing file_struct around like that
> is insane) and since the benefit is all due to the fact that it's
> *NOT* shared, *NOT* modified in parallel, etc., allowing us to
> treat file references as stable... why the hell use the descriptor
> tables at all?
This one is not a regular system call, since we don't do fget, then IO,
then fput. We hang on to it. But for the non-registered case, it's very
much just like a regular read/write system call, where we fget to do IO
on it, then fput when we are done.
> All you need is an array of struct file *, explicitly populated.
> With net/unix/garbage.c aware of such beasts. Guess what? We
> do have such an object already. The one net/unix/garbage.c is
> working with. SCM_RIGHTS datagrams, that is.
>
> IOW, can't we give those io_uring descriptors associated struct
> unix_sock? No socket descriptors, no struct socket (probably),
> just the AF_UNIX-specific part thereof. Then teach
> unix_inflight()/unix_notinflight() about getting unix_sock out
> of these guys (incidentally, both would seem to benefit from
> _not_ touching unix_gc_lock in case when there's no unix_sock
> attached to file we are dealing with - I might be missing
> something very subtle about barriers there, but it doesn't
> look likely).
That might be workable, though I'm not sure we currently have helpers to
just explicitly create a unix_sock by itself. Not familiar with the
networking bits at all, I'll take a look.
> And make that (i.e. registering the descriptors) mandatory.
I don't want to make it mandatory, that's very inflexible for managing
tons of files. The registration is useful for specific cases where we
have high frequency of operations on a set of files. Besides, it'd make
the use of the API cumbersome as well for the basic case of just wanting
to do async IO.
> Hell, combine that with creating io_uring fd, if we really
> care about the syscall count. Benefits:
We don't care about syscall count for setup as much. If you're doing
registration of a file set, you're expected to do a LOT of IO to those
files. Hence having an extra one for setup is not a concern. My concern
is just making it mandatory to do registration, I don't think that's a
workable alternative.
> * no file_struct refcount wanking
> * no fget()/fput() (conditional, at that) from kernel
> threads
> * no CLOEXEC-dependent anything; just the teardown
> on the final fput(), whichever way it comes.
> * no fun with duelling garbage collectors.
The fget/fput from a kernel thread can be solved by just hanging on to
the struct file * when we punt the IO. Right now we don't, which is a
little silly, that should be changed.
Getting rid of the files_struct{} is doable.
--
Jens Axboe
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
* Re: [PATCH 05/18] Add io_uring IO interface
From: Jens Axboe @ 2019-02-05 16:58 UTC (permalink / raw)
To: Florian Weimer; +Cc: linux-aio, linux-block, linux-api, hch, jmoyer, avi, jannh
In-Reply-To: <871s4r8j5i.fsf@oldenburg2.str.redhat.com>
On 2/1/19 11:20 AM, Florian Weimer wrote:
> * Jens Axboe:
>
>> +/*
>> + * Filled with the offset for mmap(2)
>> + */
>> +struct io_sqring_offsets {
>> + __u32 head;
>> + __u32 tail;
>> + __u32 ring_mask;
>> + __u32 ring_entries;
>> + __u32 flags;
>> + __u32 dropped;
>> + __u32 array;
>> + __u32 resv[3];
>> +};
>> +
>> +struct io_cqring_offsets {
>> + __u32 head;
>> + __u32 tail;
>> + __u32 ring_mask;
>> + __u32 ring_entries;
>> + __u32 overflow;
>> + __u32 cqes;
>> + __u32 resv[4];
>> +};
>> +
>> +/*
>> + * io_uring_enter(2) flags
>> + */
>> +#define IORING_ENTER_GETEVENTS (1U << 0)
>> +
>> +/*
>> + * Passed in for io_uring_setup(2). Copied back with updated info on success
>> + */
>> +struct io_uring_params {
>> + __u32 sq_entries;
>> + __u32 cq_entries;
>> + __u32 flags;
>> + __u32 resv[7];
>> + struct io_sqring_offsets sq_off;
>> + struct io_cqring_offsets cq_off;
>> +};
>
> I still think it might be a good idea to have one reserved __u64 field
> for alignment purposes.
Sure, I can do that.
--
Jens Axboe
--
To unsubscribe, send a message with 'unsubscribe linux-aio' in
the body to majordomo@kvack.org. For more info on Linux AIO,
see: http://www.kvack.org/aio/
Don't email: <a href=mailto:"aart@kvack.org">aart@kvack.org</a>
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox