system/linux
system/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux synced 2024-07-21 10:41:44 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

io_uring: ensure iopoll runs local task work as well

Browse source 
Combine the two checks we have for task_work running and whether or not
we need to shuffle the mutex into one, so we unify how task_work is run
in the iopoll loop. This helps ensure that local task_work is run when
needed, and also optimizes that path to avoid a mutex shuffle if it's
not needed.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2022-09-03 09:52:01 -06:00
parent 8ac5d85a89
commit dac6a0eae7
2 changed files with 23 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
io_uring/io_uring.c
View file

@ -1428,25 +1428,26 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
* forever, while the workqueue is stuck trying to acquire the
* very same mutex.
*/
if (wq_list_empty(&ctx->iopoll_list)) {
u32 tail = ctx->cached_cq_tail;
if (wq_list_empty(&ctx->iopoll_list) ||
io_task_work_pending(ctx)) {
if (!llist_empty(&ctx->work_llist))
__io_run_local_work(ctx, true);
if (task_work_pending(current) ||
wq_list_empty(&ctx->iopoll_list)) {
u32 tail = ctx->cached_cq_tail;
mutex_unlock(&ctx->uring_lock);
ret = io_run_task_work_ctx(ctx);
mutex_lock(&ctx->uring_lock);
if (ret < 0)
break;
mutex_unlock(&ctx->uring_lock);
ret = io_run_task_work();
mutex_lock(&ctx->uring_lock);
/* some requests don't go through iopoll_list */
if (tail != ctx->cached_cq_tail ||
wq_list_empty(&ctx->iopoll_list))
break;
}
if (ret < 0)
break;
if (task_work_pending(current)) {
mutex_unlock(&ctx->uring_lock);
io_run_task_work();
mutex_lock(&ctx->uring_lock);
/* some requests don't go through iopoll_list */
if (tail != ctx->cached_cq_tail ||
wq_list_empty(&ctx->iopoll_list))
break;
}
}
ret = io_do_iopoll(ctx, !min);
if (ret < 0)

6
io_uring/io_uring.h
View file

@ -236,6 +236,12 @@ static inline int io_run_task_work(void)
return 0;
}
static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
{
return test_thread_flag(TIF_NOTIFY_SIGNAL) ||
!wq_list_empty(&ctx->work_llist);
}
static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
{
int ret = 0;