io-wq: return next work from ->do_work() directly

It's easier to return next work from ->do_work() than
having an in-out argument. Looks nicer and easier to compile.
Also, merge io_wq_assign_next() into its only user.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/fs/io-wq.c b/fs/io-wq.c
index 47c5f3aeb46007eae5a031e825a45a699ace2770..72f759e1d6eb0bd2f59c2f2fe6cbbf97079f82a2 100644 (file)
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -523,9 +523,8 @@ get_next:
                                work->flags |= IO_WQ_WORK_CANCEL;
 
                        hash = io_get_work_hash(work);
-                       linked = old_work = work;
-                       wq->do_work(&linked);
-                       linked = (old_work == linked) ? NULL : linked;
+                       old_work = work;
+                       linked = wq->do_work(work);
 
                        work = next_hashed;
                        if (!work && linked && !io_wq_is_hashed(linked)) {
@@ -781,8 +780,7 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe)
                struct io_wq_work *old_work = work;
 
                work->flags |= IO_WQ_WORK_CANCEL;
-               wq->do_work(&work);
-               work = (work == old_work) ? NULL : work;
+               work = wq->do_work(work);
                wq->free_work(old_work);
        } while (work);
 }

diff --git a/fs/io-wq.h b/fs/io-wq.h
index 04239dfb12b0722c1046dbd1bf3b2ccb6711cdbe..114f12ec2d6572de57aafe8d64a17cab43fe2bd0 100644 (file)
--- a/fs/io-wq.h
+++ b/fs/io-wq.h
@@ -101,7 +101,7 @@ static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
 }
 
 typedef void (free_work_fn)(struct io_wq_work *);
-typedef void (io_wq_work_fn)(struct io_wq_work **);
+typedef struct io_wq_work *(io_wq_work_fn)(struct io_wq_work *);
 
 struct io_wq_data {
        struct user_struct *user;

diff --git a/fs/io_uring.c b/fs/io_uring.c
index b628e4429b75e06adcf9cfc1c83cac69f5530959..2e44b378826564f547102c4212e1c4cf0e9454e1 100644 (file)
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -895,7 +895,6 @@ enum io_mem_account {
        ACCT_PINNED,
 };
 
-static void io_wq_submit_work(struct io_wq_work **workptr);
 static void io_cqring_fill_event(struct io_kiocb *req, long res);
 static void io_put_req(struct io_kiocb *req);
 static void io_double_put_req(struct io_kiocb *req);
@@ -1773,20 +1772,6 @@ static void io_free_req(struct io_kiocb *req)
        }
 }
 
-static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
-{
-       struct io_kiocb *link;
-       const struct io_op_def *def = &io_op_defs[nxt->opcode];
-
-       if ((nxt->flags & REQ_F_ISREG) && def->hash_reg_file)
-               io_wq_hash_work(&nxt->work, file_inode(nxt->file));
-
-       *workptr = &nxt->work;
-       link = io_prep_linked_timeout(nxt);
-       if (link)
-               nxt->flags |= REQ_F_QUEUE_TIMEOUT;
-}
-
 /*
  * Drop reference to request, return next in chain (if there is one) if this
  * was the last reference to this request.
@@ -1806,24 +1791,29 @@ static void io_put_req(struct io_kiocb *req)
                io_free_req(req);
 }
 
-static void io_steal_work(struct io_kiocb *req,
-                         struct io_wq_work **workptr)
+static struct io_wq_work *io_steal_work(struct io_kiocb *req)
 {
+       struct io_kiocb *link, *nxt = NULL;
+
        /*
-        * It's in an io-wq worker, so there always should be at least
-        * one reference, which will be dropped in io_put_work() just
-        * after the current handler returns.
-        *
-        * It also means, that if the counter dropped to 1, then there is
-        * no asynchronous users left, so it's safe to steal the next work.
+        * A ref is owned by io-wq in which context we're. So, if that's the
+        * last one, it's safe to steal next work. False negatives are Ok,
+        * it just will be re-punted async in io_put_work()
         */
-       if (refcount_read(&req->refs) == 1) {
-               struct io_kiocb *nxt = NULL;
+       if (refcount_read(&req->refs) != 1)
+               return NULL;
 
-               io_req_find_next(req, &nxt);
-               if (nxt)
-                       io_wq_assign_next(workptr, nxt);
-       }
+       io_req_find_next(req, &nxt);
+       if (!nxt)
+               return NULL;
+
+       if ((nxt->flags & REQ_F_ISREG) && io_op_defs[nxt->opcode].hash_reg_file)
+               io_wq_hash_work(&nxt->work, file_inode(nxt->file));
+
+       link = io_prep_linked_timeout(nxt);
+       if (link)
+               nxt->flags |= REQ_F_QUEUE_TIMEOUT;
+       return &nxt->work;
 }
 
 /*
@@ -5718,9 +5708,8 @@ static void io_arm_async_linked_timeout(struct io_kiocb *req)
        io_queue_linked_timeout(link);
 }
 
-static void io_wq_submit_work(struct io_wq_work **workptr)
+static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work)
 {
-       struct io_wq_work *work = *workptr;
        struct io_kiocb *req = container_of(work, struct io_kiocb, work);
        int ret = 0;
 
@@ -5751,7 +5740,7 @@ static void io_wq_submit_work(struct io_wq_work **workptr)
                io_req_complete(req, ret);
        }
 
-       io_steal_work(req, workptr);
+       return io_steal_work(req);
 }
 
 static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,