io_uring: remove drain_active check from hot path

req->ctx->active_drain is a bit too expensive, partially because of two
dereferences. Do a trick, if we see it set in io_init_req(), set
REQ_F_FORCE_ASYNC and it automatically goes through a slower path where
we can catch it. It's nearly free to do in io_init_req() because there
is already ->restricted check and it's in the same byte of a bitmask.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/d7e7ddc63c15e8a300833132abb3eb8fd3918aef.1632516769.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 9b99ec46da2559f86f633e5ae0c3162797958966..f0fb458e81f39f72286e7c3948979e5513f36e46 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -6438,23 +6438,15 @@ static bool io_drain_req(struct io_kiocb *req)
        int ret;
        u32 seq;
 
-       if (req->flags & REQ_F_FAIL) {
-               io_req_complete_fail_submit(req);
-               return true;
-       }
-
-       /*
-        * If we need to drain a request in the middle of a link, drain the
-        * head request and the next request/link after the current link.
-        * Considering sequential execution of links, IOSQE_IO_DRAIN will be
-        * maintained for every request of our link.
-        */
-       if (ctx->drain_next) {
-               req->flags |= REQ_F_IO_DRAIN;
-               ctx->drain_next = false;
-       }
        /* not interested in head, start from the first linked */
        io_for_each_link(pos, req->link) {
+               /*
+                * If we need to drain a request in the middle of a link, drain
+                * the head request and the next request/link after the current
+                * link. Considering sequential execution of links,
+                * IOSQE_IO_DRAIN will be maintained for every request of our
+                * link.
+                */
                if (pos->flags & REQ_F_IO_DRAIN) {
                        ctx->drain_next = true;
                        req->flags |= REQ_F_IO_DRAIN;
@@ -6946,13 +6938,12 @@ issue_sqe:
 static inline void io_queue_sqe(struct io_kiocb *req)
        __must_hold(&req->ctx->uring_lock)
 {
-       if (unlikely(req->ctx->drain_active) && io_drain_req(req))
-               return;
-
        if (likely(!(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))) {
                __io_queue_sqe(req);
        } else if (req->flags & REQ_F_FAIL) {
                io_req_complete_fail_submit(req);
+       } else if (unlikely(req->ctx->drain_active) && io_drain_req(req)) {
+               return;
        } else {
                int ret = io_req_prep_async(req);
 
@@ -6972,9 +6963,6 @@ static inline bool io_check_restriction(struct io_ring_ctx *ctx,
                                        struct io_kiocb *req,
                                        unsigned int sqe_flags)
 {
-       if (likely(!ctx->restricted))
-               return true;
-
        if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
                return false;
 
@@ -7015,11 +7003,28 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
                if ((sqe_flags & IOSQE_BUFFER_SELECT) &&
                    !io_op_defs[req->opcode].buffer_select)
                        return -EOPNOTSUPP;
-               if (sqe_flags & IOSQE_IO_DRAIN)
+               if (sqe_flags & IOSQE_IO_DRAIN) {
+                       struct io_submit_link *link = &ctx->submit_state.link;
+
                        ctx->drain_active = true;
+                       req->flags |= REQ_F_FORCE_ASYNC;
+                       if (link->head)
+                               link->head->flags |= IOSQE_IO_DRAIN | REQ_F_FORCE_ASYNC;
+               }
+       }
+       if (unlikely(ctx->restricted || ctx->drain_active || ctx->drain_next)) {
+               if (ctx->restricted && !io_check_restriction(ctx, req, sqe_flags))
+                       return -EACCES;
+               /* knock it to the slow queue path, will be drained there */
+               if (ctx->drain_active)
+                       req->flags |= REQ_F_FORCE_ASYNC;
+               /* if there is no link, we're at "next" request and need to drain */
+               if (unlikely(ctx->drain_next) && !ctx->submit_state.link.head) {
+                       ctx->drain_next = false;
+                       ctx->drain_active = true;
+                       req->flags |= REQ_F_FORCE_ASYNC | IOSQE_IO_DRAIN;
+               }
        }
-       if (!io_check_restriction(ctx, req, sqe_flags))
-               return -EACCES;
 
        personality = READ_ONCE(sqe->personality);
        if (personality) {