diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2021-10-06 16:06:49 +0100 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2021-10-19 05:49:55 -0600 |
| commit | 4a04d1d14831d31f2cd0e31eb1568cc9c1be0095 (patch) | |
| tree | 519b920651e76343cb0af5abb3822a3a583356e7 | |
| parent | 5a158c6b0d033893cc80c28b182e1207253768a5 (diff) | |
io_uring: optimise out req->opcode reloading
Looking at the assembly, the compiler decided to reload req->opcode in
io_op_defs[opcode].needs_file instead of one it had in a register, so
store it in a temp variable so it can be optimised out. Also move the
personality block later, it's better for spilling/etc. as it only
depends on @sqe, which we're keeping anyway.
By the way, zero req->opcode if it over IORING_OP_LAST, not a problem,
at the moment but is safer.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/6ba869f5f8b7b0f991c87fdf089f0abf87cbe06b.1633532552.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
| -rw-r--r-- | fs/io_uring.c | 31 |
1 files changed, 17 insertions, 14 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c index 04d6e35ea0df..7918a320104d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6975,9 +6975,10 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, { unsigned int sqe_flags; int personality; + u8 opcode; /* req is partially pre-initialised, see io_preinit_req() */ - req->opcode = READ_ONCE(sqe->opcode); + req->opcode = opcode = READ_ONCE(sqe->opcode); /* same numerical values with corresponding REQ_F_*, safe to copy */ req->flags = sqe_flags = READ_ONCE(sqe->flags); req->user_data = READ_ONCE(sqe->user_data); @@ -6985,14 +6986,16 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, req->fixed_rsrc_refs = NULL; req->task = current; - if (unlikely(req->opcode >= IORING_OP_LAST)) + if (unlikely(opcode >= IORING_OP_LAST)) { + req->opcode = 0; return -EINVAL; + } if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) { /* enforce forwards compatibility on users */ if (sqe_flags & ~SQE_VALID_FLAGS) return -EINVAL; if ((sqe_flags & IOSQE_BUFFER_SELECT) && - !io_op_defs[req->opcode].buffer_select) + !io_op_defs[opcode].buffer_select) return -EOPNOTSUPP; if (sqe_flags & IOSQE_IO_DRAIN) io_init_req_drain(req); @@ -7011,23 +7014,14 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, } } - personality = READ_ONCE(sqe->personality); - if (personality) { - req->creds = xa_load(&ctx->personalities, personality); - if (!req->creds) - return -EINVAL; - get_cred(req->creds); - req->flags |= REQ_F_CREDS; - } - - if (io_op_defs[req->opcode].needs_file) { + if (io_op_defs[opcode].needs_file) { struct io_submit_state *state = &ctx->submit_state; /* * Plug now if we have more than 2 IO left after this, and the * target is potentially a read/write to block based storage. */ - if (state->need_plug && io_op_defs[req->opcode].plug) { + if (state->need_plug && io_op_defs[opcode].plug) { state->plug_started = true; state->need_plug = false; blk_start_plug(&state->plug); @@ -7039,6 +7033,15 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, return -EBADF; } + personality = READ_ONCE(sqe->personality); + if (personality) { + req->creds = xa_load(&ctx->personalities, personality); + if (!req->creds) + return -EINVAL; + get_cred(req->creds); + req->flags |= REQ_F_CREDS; + } + return io_req_prep(req, sqe); } |