io_uring: modify io_get_cqe for CQE32

Modify accesses to the CQE array to take large CQE's into account. The
index needs to be shifted by one for large CQE's.

Signed-off-by: Stefan Roesch <shr@fb.com>
Reviewed-by: Kanchan Joshi <joshi.k@samsung.com>
Link: https://lore.kernel.org/r/20220426182134.136504-7-shr@fb.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 5ec7a4b82aa5c5f6821e25a93b038cf632d7493d..3915251bfecabe5c9f4b9fa120d6221020d842ea 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2069,8 +2069,12 @@ static noinline struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)
 {
        struct io_rings *rings = ctx->rings;
        unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
+       unsigned int shift = 0;
        unsigned int free, queued, len;
 
+       if (ctx->flags & IORING_SETUP_CQE32)
+               shift = 1;
+
        /* userspace may cheat modifying the tail, be safe and do min */
        queued = min(__io_cqring_events(ctx), ctx->cq_entries);
        free = ctx->cq_entries - queued;
@@ -2082,15 +2086,26 @@ static noinline struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)
        ctx->cached_cq_tail++;
        ctx->cqe_cached = &rings->cqes[off];
        ctx->cqe_sentinel = ctx->cqe_cached + len;
-       return ctx->cqe_cached++;
+       ctx->cqe_cached++;
+       return &rings->cqes[off << shift];
 }
 
 static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
 {
        if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {
+               struct io_uring_cqe *cqe = ctx->cqe_cached;
+
+               if (ctx->flags & IORING_SETUP_CQE32) {
+                       unsigned int off = ctx->cqe_cached - ctx->rings->cqes;
+
+                       cqe += off;
+               }
+
                ctx->cached_cq_tail++;
-               return ctx->cqe_cached++;
+               ctx->cqe_cached++;
+               return cqe;
        }
+
        return __io_get_cqe(ctx);
 }