io_uring: refactor io_req_complete_post()

Make io_req_complete_post() to push all IORING_SETUP_IOPOLL requests
to task_work, it's much cleaner and should normally happen. We couldn't
do it before because there was a possibility of looping in

complete_post() -> tw -> complete_post() -> ...

Also, unexport the function and inline __io_req_complete_post().

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Tested-by: Ming Lei <ming.lei@redhat.com>
Link: https://lore.kernel.org/r/ea19c032ace3e0dd96ac4d991a063b0188037014.1710799188.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index a0073625ff163bdf5b39225ad10b5b6182ba3247..951ff3b787abfc086ee10701fe5b00a3bd38e929 100644 (file)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -920,11 +920,21 @@ bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags)
        return posted;
 }
 
-static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
+static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 {
        struct io_ring_ctx *ctx = req->ctx;
        struct io_rsrc_node *rsrc_node = NULL;
 
+       /*
+        * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
+        * the submitter task context, IOPOLL protects with uring_lock.
+        */
+       if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL)) {
+               req->io_task_work.func = io_req_task_complete;
+               io_req_task_work_add(req);
+               return;
+       }
+
        io_cq_lock(ctx);
        if (!(req->flags & REQ_F_CQE_SKIP)) {
                if (!io_fill_cqe_req(ctx, req))
@@ -968,23 +978,6 @@ static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
        }
 }
 
-void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
-{
-       struct io_ring_ctx *ctx = req->ctx;
-
-       if (ctx->task_complete) {
-               req->io_task_work.func = io_req_task_complete;
-               io_req_task_work_add(req);
-       } else if (!(issue_flags & IO_URING_F_UNLOCKED) ||
-                  !(ctx->flags & IORING_SETUP_IOPOLL)) {
-               __io_req_complete_post(req, issue_flags);
-       } else {
-               mutex_lock(&ctx->uring_lock);
-               __io_req_complete_post(req, issue_flags & ~IO_URING_F_UNLOCKED);
-               mutex_unlock(&ctx->uring_lock);
-       }
-}
-
 void io_req_defer_failed(struct io_kiocb *req, s32 res)
        __must_hold(&ctx->uring_lock)
 {

diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 5119265a11c217378afc6a1b8d26ccbba71076f7..f694e7e6fb255ea53bedd167d87e255f809449b9 100644 (file)
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -65,7 +65,6 @@ bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow);
 void io_req_cqe_overflow(struct io_kiocb *req);
 int io_run_task_work_sig(struct io_ring_ctx *ctx);
 void io_req_defer_failed(struct io_kiocb *req, s32 res);
-void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags);
 bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags);
 bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags);
 void __io_commit_cqring_flush(struct io_ring_ctx *ctx);