niansa-misc/linux
1
0
Fork 0
mirror of synced 2025-03-06 20:59:54 +01:00
Code Issues Projects Releases Packages Wiki Activity

io_uring: complete all requests in task context

Browse source 
This patch adds ctx->task_complete flag. If set, we'll complete all
requests in the context of the original task. Note, this extends to
completion CQE posting only but not io_kiocb cleanup / free, e.g. io-wq
may free the requests in the free calllback. This flag will be used
later for optimisations purposes.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/21ece72953f76bb2e77659a72a14326227ab6460.1670384893.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2022-12-07 03:53:30 +00:00 • committed by Jens Axboe
parent 1b346e4aa8
commit e6aeb2721d
3 changed files with 15 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
include/linux/io_uring.h
View file

@ -11,6 +11,8 @@ enum io_uring_cmd_flags {
IO_URING_F_UNLOCKED = 2, IO_URING_F_UNLOCKED = 2,
/* the request is executed from poll, it should not be freed */ /* the request is executed from poll, it should not be freed */
IO_URING_F_MULTISHOT = 4, IO_URING_F_MULTISHOT = 4,
/* executed by io-wq */
IO_URING_F_IOWQ = 8,
/* int's last bit, sign checks are usually faster than a bit test */ /* int's last bit, sign checks are usually faster than a bit test */
IO_URING_F_NONBLOCK = INT_MIN, IO_URING_F_NONBLOCK = INT_MIN,

2
include/linux/io_uring_types.h
View file

@ -208,6 +208,8 @@ struct io_ring_ctx {
unsigned int drain_disabled: 1; unsigned int drain_disabled: 1;
unsigned int has_evfd: 1; unsigned int has_evfd: 1;
unsigned int syscall_iopoll: 1; unsigned int syscall_iopoll: 1;
/* all CQEs should be posted only by the submitter task */
unsigned int task_complete: 1;
} ____cacheline_aligned_in_smp; } ____cacheline_aligned_in_smp;
/* submission data */ /* submission data */

12
io_uring/io_uring.c
View file

@ -921,7 +921,10 @@ static void __io_req_complete_post(struct io_kiocb *req)
void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
{ {
if (!(issue_flags & IO_URING_F_UNLOCKED) || if (req->ctx->task_complete && (issue_flags & IO_URING_F_IOWQ)) {
req->io_task_work.func = io_req_task_complete;
io_req_task_work_add(req);
} else if (!(issue_flags & IO_URING_F_UNLOCKED) ||
!(req->ctx->flags & IORING_SETUP_IOPOLL)) { !(req->ctx->flags & IORING_SETUP_IOPOLL)) {
__io_req_complete_post(req); __io_req_complete_post(req);
} else { } else {
@ -1830,7 +1833,7 @@ void io_wq_submit_work(struct io_wq_work *work)
{ {
struct io_kiocb *req = container_of(work, struct io_kiocb, work); struct io_kiocb *req = container_of(work, struct io_kiocb, work);
const struct io_op_def *def = &io_op_defs[req->opcode]; const struct io_op_def *def = &io_op_defs[req->opcode];
unsigned int issue_flags = IO_URING_F_UNLOCKED; unsigned int issue_flags = IO_URING_F_UNLOCKED | IO_URING_F_IOWQ;
bool needs_poll = false; bool needs_poll = false;
int ret = 0, err = -ECANCELED; int ret = 0, err = -ECANCELED;
@ -3490,6 +3493,11 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
if (!ctx) if (!ctx)
return -ENOMEM; return -ENOMEM;
if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
!(ctx->flags & IORING_SETUP_IOPOLL) &&
!(ctx->flags & IORING_SETUP_SQPOLL))
ctx->task_complete = true;
/* /*
* When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
* space applications don't need to do io completion events * space applications don't need to do io completion events