diff options
author | Pavel Begunkov <asml.silence@gmail.com> | 2021-10-04 20:02:49 +0100 |
---|---|---|
committer | Jens Axboe <axboe@kernel.dk> | 2021-10-19 05:49:54 -0600 |
commit | a33ae9ce16a8ca62c5dffbe8909d185c6c5b4d77 (patch) | |
tree | 15dbe6fca6f7bd0a23a6d99a12a10b6e257af760 | |
parent | fff4e40e3094972b119e38c8f0de4e1ca9fec654 (diff) |
io_uring: optimise request allocation
Even after fully inlining io_alloc_req() my compiler does a NULL check
in the path of successful allocation, no hacks like an empty dereference
help it. Restructure io_alloc_req() by splitting out refilling part, so
the compiler generate a slightly better binary.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/eda17571bdc7248d8e617b23e7132a5416e4680b.1633373302.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
-rw-r--r-- | fs/io_uring.c | 28 |
1 files changed, 20 insertions, 8 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c index 045474b7b9c2..614c982c2697 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1912,18 +1912,17 @@ static bool io_flush_cached_reqs(struct io_ring_ctx *ctx) * Because of that, io_alloc_req() should be called only under ->uring_lock * and with extra caution to not get a request that is still worked on. */ -static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) +static bool __io_alloc_req_refill(struct io_ring_ctx *ctx) __must_hold(&ctx->uring_lock) { struct io_submit_state *state = &ctx->submit_state; gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; void *reqs[IO_REQ_ALLOC_BATCH]; - struct io_wq_work_node *node; struct io_kiocb *req; int ret, i; if (likely(state->free_list.next || io_flush_cached_reqs(ctx))) - goto got_req; + return true; ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs); @@ -1934,7 +1933,7 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) if (unlikely(ret <= 0)) { reqs[0] = kmem_cache_alloc(req_cachep, gfp); if (!reqs[0]) - return NULL; + return false; ret = 1; } @@ -1944,8 +1943,21 @@ static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) io_preinit_req(req, ctx); wq_stack_add_head(&req->comp_list, &state->free_list); } -got_req: - node = wq_stack_extract(&state->free_list); + return true; +} + +static inline bool io_alloc_req_refill(struct io_ring_ctx *ctx) +{ + if (unlikely(!ctx->submit_state.free_list.next)) + return __io_alloc_req_refill(ctx); + return true; +} + +static inline struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx) +{ + struct io_wq_work_node *node; + + node = wq_stack_extract(&ctx->submit_state.free_list); return container_of(node, struct io_kiocb, comp_list); } @@ -7217,12 +7229,12 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) const struct io_uring_sqe *sqe; struct io_kiocb *req; - req = io_alloc_req(ctx); - if (unlikely(!req)) { + if (unlikely(!io_alloc_req_refill(ctx))) { if (!submitted) submitted = -EAGAIN; break; } + req = io_alloc_req(ctx); sqe = io_get_sqe(ctx); if (unlikely(!sqe)) { wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list); |