system/linux
system/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux synced 2024-07-21 10:41:44 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

io_uring: add rsrc referencing for notifiers

Browse source 
In preparation to zerocopy sends with fixed buffers make notifiers to
reference the rsrc node to protect the used fixed buffers. We can't just
grab it for a send request as notifiers can likely outlive requests that
used it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/3cd7a01d26837945b6982fa9cf15a63230f2ed4f.1657643355.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2022-07-12 21:52:41 +01:00 committed by Jens Axboe
parent e58d498e81
commit 68ef5578ef
3 changed files with 15 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
io_uring/notif.c
View file

@ -7,10 +7,12 @@
#include "io_uring.h"
#include "notif.h"
#include "rsrc.h"
static void __io_notif_complete_tw(struct callback_head *cb)
{
struct io_notif *notif = container_of(cb, struct io_notif, task_work);
struct io_rsrc_node *rsrc_node = notif->rsrc_node;
struct io_ring_ctx *ctx = notif->ctx;
if (likely(notif->task)) {
@ -25,6 +27,7 @@ static void __io_notif_complete_tw(struct callback_head *cb)
ctx->notif_locked_nr++;
io_cq_unlock_post(ctx);
io_rsrc_put_node(rsrc_node, 1);
percpu_ref_put(&ctx->refs);
}
@ -119,6 +122,8 @@ struct io_notif *io_alloc_notif(struct io_ring_ctx *ctx,
/* master ref owned by io_notif_slot, will be dropped on flush */
refcount_set(&notif->uarg.refcnt, 1);
percpu_ref_get(&ctx->refs);
notif->rsrc_node = ctx->rsrc_node;
io_charge_rsrc_node(ctx);
return notif;
}

1
io_uring/notif.h
View file

@ -10,6 +10,7 @@
struct io_notif {
struct ubuf_info uarg;
struct io_ring_ctx *ctx;
struct io_rsrc_node *rsrc_node;
/* complete via tw if ->task is non-NULL, fallback to wq otherwise */
struct task_struct *task;

12
io_uring/rsrc.h
View file

@ -135,6 +135,13 @@ static inline void io_req_put_rsrc_locked(struct io_kiocb *req,
}
}
static inline void io_charge_rsrc_node(struct io_ring_ctx *ctx)
{
ctx->rsrc_cached_refs--;
if (unlikely(ctx->rsrc_cached_refs < 0))
io_rsrc_refs_refill(ctx);
}
static inline void io_req_set_rsrc_node(struct io_kiocb *req,
struct io_ring_ctx *ctx,
unsigned int issue_flags)
@ -144,9 +151,8 @@ static inline void io_req_set_rsrc_node(struct io_kiocb *req,
if (!(issue_flags & IO_URING_F_UNLOCKED)) {
lockdep_assert_held(&ctx->uring_lock);
ctx->rsrc_cached_refs--;
if (unlikely(ctx->rsrc_cached_refs < 0))
io_rsrc_refs_refill(ctx);
io_charge_rsrc_node(ctx);
} else {
percpu_ref_get(&req->rsrc_node->refs);
}