mirror of
https://github.com/torvalds/linux
synced 2024-09-06 18:01:05 +00:00
io_uring/rsrc: improve code generation for fixed file assignment
For the normal read/write path, we have already locked the ring submission side when assigning the file. This causes branch mispredictions when we then check and try and lock again in io_req_set_rsrc_node(). As this is a very hot path, this matters. Add a basic helper that already assumes we already have it locked, and use that in io_file_get_fixed(). Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
parent
fe80eb15de
commit
3f302388d4
|
@ -2000,9 +2000,10 @@ inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
|
||||||
goto out;
|
goto out;
|
||||||
fd = array_index_nospec(fd, ctx->nr_user_files);
|
fd = array_index_nospec(fd, ctx->nr_user_files);
|
||||||
slot = io_fixed_file_slot(&ctx->file_table, fd);
|
slot = io_fixed_file_slot(&ctx->file_table, fd);
|
||||||
file = io_slot_file(slot);
|
if (!req->rsrc_node)
|
||||||
|
__io_req_set_rsrc_node(req, ctx);
|
||||||
req->flags |= io_slot_flags(slot);
|
req->flags |= io_slot_flags(slot);
|
||||||
io_req_set_rsrc_node(req, ctx, 0);
|
file = io_slot_file(slot);
|
||||||
out:
|
out:
|
||||||
io_ring_submit_unlock(ctx, issue_flags);
|
io_ring_submit_unlock(ctx, issue_flags);
|
||||||
return file;
|
return file;
|
||||||
|
|
|
@ -102,17 +102,21 @@ static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx,
|
||||||
node->refs++;
|
node->refs++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static inline void __io_req_set_rsrc_node(struct io_kiocb *req,
|
||||||
|
struct io_ring_ctx *ctx)
|
||||||
|
{
|
||||||
|
lockdep_assert_held(&ctx->uring_lock);
|
||||||
|
req->rsrc_node = ctx->rsrc_node;
|
||||||
|
io_charge_rsrc_node(ctx, ctx->rsrc_node);
|
||||||
|
}
|
||||||
|
|
||||||
static inline void io_req_set_rsrc_node(struct io_kiocb *req,
|
static inline void io_req_set_rsrc_node(struct io_kiocb *req,
|
||||||
struct io_ring_ctx *ctx,
|
struct io_ring_ctx *ctx,
|
||||||
unsigned int issue_flags)
|
unsigned int issue_flags)
|
||||||
{
|
{
|
||||||
if (!req->rsrc_node) {
|
if (!req->rsrc_node) {
|
||||||
io_ring_submit_lock(ctx, issue_flags);
|
io_ring_submit_lock(ctx, issue_flags);
|
||||||
|
__io_req_set_rsrc_node(req, ctx);
|
||||||
lockdep_assert_held(&ctx->uring_lock);
|
|
||||||
|
|
||||||
req->rsrc_node = ctx->rsrc_node;
|
|
||||||
io_charge_rsrc_node(ctx, ctx->rsrc_node);
|
|
||||||
io_ring_submit_unlock(ctx, issue_flags);
|
io_ring_submit_unlock(ctx, issue_flags);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue