io_uring: move submit side state closer in the ring

We recently added the submit side req cache, but it was placed at the
end of the struct. Move it near the other submission state for better
memory placement, and reshuffle a few other members at the same time.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io_uring.c b/fs/io_uring.c
index cd9c4c05f6f5f4a7fa1b4dd87a25d9d7064141db..8be7a24aa10ebcd9253b8e55f1060c25933ae838 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -346,6 +346,13 @@ struct io_ring_ctx {
                struct io_uring_sqe     *sq_sqes;
        } ____cacheline_aligned_in_smp;
 
+       struct {
+               struct mutex            uring_lock;
+               wait_queue_head_t       wait;
+       } ____cacheline_aligned_in_smp;
+
+       struct io_submit_state          submit_state;
+
        struct io_rings *rings;
 
        /* IO offload */
@@ -413,11 +420,6 @@ struct io_ring_ctx {
                struct eventfd_ctx      *cq_ev_fd;
        } ____cacheline_aligned_in_smp;
 
-       struct {
-               struct mutex            uring_lock;
-               wait_queue_head_t       wait;
-       } ____cacheline_aligned_in_smp;
-
        struct {
                spinlock_t              completion_lock;
 
@@ -441,9 +443,10 @@ struct io_ring_ctx {
        struct list_head                rsrc_ref_list;
        spinlock_t                      rsrc_ref_lock;
 
-       struct work_struct              exit_work;
        struct io_restriction           restrictions;
-       struct io_submit_state          submit_state;
+
+       /* Keep this last, we don't need it for the fast path */
+       struct work_struct              exit_work;
 };
 
 /*