io_uring: refactor provide/remove buffer locking

Always complete request holding the mutex instead of doing that strange
dancing with conditional ordering.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 11a02ec86e54da56b5cd339f6d9090820e22e61e..2642369d1332111a46fffce19ab97e573fdfbaae 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -3966,14 +3966,9 @@ static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
        if (ret < 0)
                req_set_fail_links(req);
 
-       /* need to hold the lock to complete IOPOLL requests */
-       if (ctx->flags & IORING_SETUP_IOPOLL) {
-               __io_req_complete(req, issue_flags, ret, 0);
-               io_ring_submit_unlock(ctx, !force_nonblock);
-       } else {
-               io_ring_submit_unlock(ctx, !force_nonblock);
-               __io_req_complete(req, issue_flags, ret, 0);
-       }
+       /* complete before unlock, IOPOLL may need the lock */
+       __io_req_complete(req, issue_flags, ret, 0);
+       io_ring_submit_unlock(ctx, !force_nonblock);
        return 0;
 }
 
@@ -4055,15 +4050,9 @@ static int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags)
        }
        if (ret < 0)
                req_set_fail_links(req);
-
-       /* need to hold the lock to complete IOPOLL requests */
-       if (ctx->flags & IORING_SETUP_IOPOLL) {
-               __io_req_complete(req, issue_flags, ret, 0);
-               io_ring_submit_unlock(ctx, !force_nonblock);
-       } else {
-               io_ring_submit_unlock(ctx, !force_nonblock);
-               __io_req_complete(req, issue_flags, ret, 0);
-       }
+       /* complete before unlock, IOPOLL may need the lock */
+       __io_req_complete(req, issue_flags, ret, 0);
+       io_ring_submit_unlock(ctx, !force_nonblock);
        return 0;
 }