bcachefs: More dio inlining

Eliminate another function call in the O_DIRECT write path.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c
index 8deb476a17c8060509d2618480178c5970084b5d..4dd5ebafe74259bc7985877a09e33afa88e92232 100644 (file)
--- a/fs/bcachefs/fs-io.c
+++ b/fs/bcachefs/fs-io.c
@@ -2218,7 +2218,7 @@ static __always_inline void bch2_dio_write_end(struct dio_write *dio)
                set_bit(EI_INODE_ERROR, &inode->ei_flags);
 }
 
-static long bch2_dio_write_loop(struct dio_write *dio)
+static __always_inline long bch2_dio_write_loop(struct dio_write *dio)
 {
        struct bch_fs *c = dio->op.c;
        struct kiocb *req = dio->req;
@@ -2333,18 +2333,10 @@ err:
        goto out;
 }
 
-static void bch2_dio_write_loop_async(struct bch_write_op *op)
+static noinline __cold void bch2_dio_write_continue(struct dio_write *dio)
 {
-       struct dio_write *dio = container_of(op, struct dio_write, op);
        struct mm_struct *mm = dio->mm;
 
-       bch2_dio_write_end(dio);
-
-       if (likely(!dio->iter.count) || dio->op.error) {
-               bch2_dio_write_done(dio);
-               return;
-       }
-
        bio_reset(&dio->op.wbio.bio, NULL, REQ_OP_WRITE);
 
        if (mm)
@@ -2354,6 +2346,18 @@ static void bch2_dio_write_loop_async(struct bch_write_op *op)
                kthread_unuse_mm(mm);
 }
 
+static void bch2_dio_write_loop_async(struct bch_write_op *op)
+{
+       struct dio_write *dio = container_of(op, struct dio_write, op);
+
+       bch2_dio_write_end(dio);
+
+       if (likely(!dio->iter.count) || dio->op.error)
+               bch2_dio_write_done(dio);
+       else
+               bch2_dio_write_continue(dio);
+}
+
 static noinline
 ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter)
 {