system/linux
system/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux synced 2024-09-16 09:05:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

io_uring/rsrc: optimise io_rsrc_put allocation

Browse source 
Every io_rsrc_node keeps a list of items to put, and all entries are
kmalloc()'ed. However, it's quite often to queue up only one entry per
node, so let's add an inline entry there to avoid extra allocations.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/c482c1c652c45c85ac52e67c974bc758a50fed5f.1680576071.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2023-04-04 13:39:52 +01:00 committed by Jens Axboe
parent c824986c11
commit ff7c75ecaa
2 changed files with 36 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
io_uring/rsrc.c
View file

@ -140,26 +140,34 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slo
*slot = NULL;
}
static void io_rsrc_put_work_one(struct io_rsrc_data *rsrc_data,
struct io_rsrc_put *prsrc)
{
struct io_ring_ctx *ctx = rsrc_data->ctx;
if (prsrc->tag) {
if (ctx->flags & IORING_SETUP_IOPOLL) {
mutex_lock(&ctx->uring_lock);
io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
mutex_unlock(&ctx->uring_lock);
} else {
io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
}
}
rsrc_data->do_put(ctx, prsrc);
}
static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
{
struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
struct io_ring_ctx *ctx = rsrc_data->ctx;
struct io_rsrc_put *prsrc, *tmp;
if (ref_node->inline_items)
io_rsrc_put_work_one(rsrc_data, &ref_node->item);
list_for_each_entry_safe(prsrc, tmp, &ref_node->item_list, list) {
list_del(&prsrc->list);
if (prsrc->tag) {
if (ctx->flags & IORING_SETUP_IOPOLL) {
mutex_lock(&ctx->uring_lock);
io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
mutex_unlock(&ctx->uring_lock);
} else {
io_post_aux_cqe(ctx, prsrc->tag, 0, 0);
}
}
rsrc_data->do_put(ctx, prsrc);
io_rsrc_put_work_one(rsrc_data, prsrc);
kfree(prsrc);
}
@ -251,6 +259,7 @@ static struct io_rsrc_node *io_rsrc_node_alloc(void)
INIT_LIST_HEAD(&ref_node->node);
INIT_LIST_HEAD(&ref_node->item_list);
ref_node->done = false;
ref_node->inline_items = 0;
return ref_node;
}
@ -729,15 +738,23 @@ int io_queue_rsrc_removal(struct io_rsrc_data *data, unsigned idx,
{
u64 *tag_slot = io_get_tag_slot(data, idx);
struct io_rsrc_put *prsrc;
bool inline_item = true;
prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
if (!prsrc)
return -ENOMEM;
if (!node->inline_items) {
prsrc = &node->item;
node->inline_items++;
} else {
prsrc = kzalloc(sizeof(*prsrc), GFP_KERNEL);
if (!prsrc)
return -ENOMEM;
inline_item = false;
}
prsrc->tag = *tag_slot;
*tag_slot = 0;
prsrc->rsrc = rsrc;
list_add(&prsrc->list, &node->item_list);
if (!inline_item)
list_add(&prsrc->list, &node->item_list);
return 0;
}

2
io_uring/rsrc.h
View file

@ -49,6 +49,8 @@ struct io_rsrc_node {
* came from the same table and so are of the same type.
*/
struct list_head item_list;
struct io_rsrc_put item;
int inline_items;
};
struct io_mapped_ubuf {