bcachefs: Inline more of bch2_trans_commit hot path

The main optimization here is that if we let
bch2_replicas_delta_list_apply() fail, we can completely skip calling
bch2_bkey_replicas_marked_locked().

And assorted other small optimizations.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c
index 25ad6b69b6bdbcae7ea115ae9e2baa74b3b8ce68..c264b927f992bbcc46c4af31ba4d2e947b9f7afe 100644 (file)
--- a/fs/bcachefs/btree_iter.c
+++ b/fs/bcachefs/btree_iter.c
@@ -64,21 +64,9 @@ static inline int btree_iter_pos_cmp(struct btree_iter *iter,
 
 /* Btree node locking: */
 
-/*
- * Updates the saved lock sequence number, so that bch2_btree_node_relock() will
- * succeed:
- */
 void bch2_btree_node_unlock_write(struct btree *b, struct btree_iter *iter)
 {
-       struct btree_iter *linked;
-
-       EBUG_ON(iter->l[b->c.level].b != b);
-       EBUG_ON(iter->l[b->c.level].lock_seq + 1 != b->c.lock.state.seq);
-
-       trans_for_each_iter_with_node(iter->trans, b, linked)
-               linked->l[b->c.level].lock_seq += 2;
-
-       six_unlock_write(&b->c.lock);
+       bch2_btree_node_unlock_write_inlined(b, iter);
 }
 
 void __bch2_btree_node_lock_write(struct btree *b, struct btree_iter *iter)

diff --git a/fs/bcachefs/btree_locking.h b/fs/bcachefs/btree_locking.h
index aa5882cc485285602e10c66f3510c3e88d59121b..a164924ca656e51b8f865a0b5f273b9d1ad3d4b1 100644 (file)
--- a/fs/bcachefs/btree_locking.h
+++ b/fs/bcachefs/btree_locking.h
@@ -202,6 +202,24 @@ static inline bool bch2_btree_node_relock(struct btree_iter *iter,
                __bch2_btree_node_relock(iter, level);
 }
 
+/*
+ * Updates the saved lock sequence number, so that bch2_btree_node_relock() will
+ * succeed:
+ */
+static inline void
+bch2_btree_node_unlock_write_inlined(struct btree *b, struct btree_iter *iter)
+{
+       struct btree_iter *linked;
+
+       EBUG_ON(iter->l[b->c.level].b != b);
+       EBUG_ON(iter->l[b->c.level].lock_seq + 1 != b->c.lock.state.seq);
+
+       trans_for_each_iter_with_node(iter->trans, b, linked)
+               linked->l[b->c.level].lock_seq += 2;
+
+       six_unlock_write(&b->c.lock);
+}
+
 void bch2_btree_node_unlock_write(struct btree *, struct btree_iter *);
 
 void __bch2_btree_node_lock_write(struct btree *, struct btree_iter *);

diff --git a/fs/bcachefs/btree_update_interior.h b/fs/bcachefs/btree_update_interior.h
index f9e092bf69aa5e1c1407c39ba4ff3804064f98b6..85f1320fa7b125692b0c969e4c4f8ce0dfc30308 100644 (file)
--- a/fs/bcachefs/btree_update_interior.h
+++ b/fs/bcachefs/btree_update_interior.h
@@ -284,17 +284,17 @@ static inline unsigned btree_write_set_buffer(struct btree *b)
 static inline struct btree_node_entry *want_new_bset(struct bch_fs *c,
                                                     struct btree *b)
 {
-       struct bset *i = btree_bset_last(b);
+       struct bset_tree *t = bset_tree_last(b);
        struct btree_node_entry *bne = max(write_block(b),
                        (void *) btree_bkey_last(b, bset_tree_last(b)));
        ssize_t remaining_space =
                __bch_btree_u64s_remaining(c, b, &bne->keys.start[0]);
 
-       if (unlikely(bset_written(b, i))) {
+       if (unlikely(bset_written(b, bset(b, t)))) {
                if (remaining_space > (ssize_t) (block_bytes(c) >> 3))
                        return bne;
        } else {
-               if (unlikely(vstruct_bytes(i) > btree_write_set_buffer(b)) &&
+               if (unlikely(bset_u64s(t) * sizeof(u64) > btree_write_set_buffer(b)) &&
                    remaining_space > (ssize_t) (btree_write_set_buffer(b) >> 3))
                        return bne;
        }

diff --git a/fs/bcachefs/btree_update_leaf.c b/fs/bcachefs/btree_update_leaf.c
index 38a27d3a3b400ec0b88ada156819bd4ceee8c5aa..85580e63b5cad739bf8617fcfab9dc84e9764965 100644 (file)
--- a/fs/bcachefs/btree_update_leaf.c
+++ b/fs/bcachefs/btree_update_leaf.c
@@ -17,6 +17,7 @@
 #include "replicas.h"
 #include "trace.h"
 
+#include <linux/prefetch.h>
 #include <linux/sort.h>
 
 static inline bool same_leaf_as_prev(struct btree_trans *trans,
@@ -50,23 +51,6 @@ inline void bch2_btree_node_lock_for_insert(struct bch_fs *c, struct btree *b,
                bch2_btree_init_next(c, b, iter);
 }
 
-static void btree_trans_lock_write(struct btree_trans *trans, bool lock)
-{
-       struct bch_fs *c = trans->c;
-       struct btree_insert_entry *i;
-       unsigned iter;
-
-       trans_for_each_update_sorted(trans, i, iter) {
-               if (same_leaf_as_prev(trans, iter))
-                       continue;
-
-               if (lock)
-                       bch2_btree_node_lock_for_insert(c, i->iter->l[0].b, i->iter);
-               else
-                       bch2_btree_node_unlock_write(i->iter->l[0].b, i->iter);
-       }
-}
-
 static inline void btree_trans_sort_updates(struct btree_trans *trans)
 {
        struct btree_insert_entry *l, *r;
@@ -377,29 +361,6 @@ btree_key_can_insert(struct btree_trans *trans,
        return BTREE_INSERT_OK;
 }
 
-static int btree_trans_check_can_insert(struct btree_trans *trans,
-                                       struct btree_insert_entry **stopped_at)
-{
-       struct btree_insert_entry *i;
-       unsigned iter, u64s = 0;
-       int ret;
-
-       trans_for_each_update_sorted(trans, i, iter) {
-               /* Multiple inserts might go to same leaf: */
-               if (!same_leaf_as_prev(trans, iter))
-                       u64s = 0;
-
-               u64s += i->k->k.u64s;
-               ret = btree_key_can_insert(trans, i, &u64s);
-               if (ret) {
-                       *stopped_at = i;
-                       return ret;
-               }
-       }
-
-       return 0;
-}
-
 static inline void do_btree_insert_one(struct btree_trans *trans,
                                       struct btree_insert_entry *insert)
 {
@@ -450,6 +411,8 @@ bch2_trans_commit_write_locked(struct btree_trans *trans,
        unsigned mark_flags = trans->flags & BTREE_INSERT_BUCKET_INVALIDATE
                ? BCH_BUCKET_MARK_BUCKET_INVALIDATE
                : 0;
+       unsigned iter, u64s = 0;
+       bool marking = false;
        int ret;
 
        if (race_fault()) {
@@ -462,25 +425,28 @@ bch2_trans_commit_write_locked(struct btree_trans *trans,
         * held, otherwise another thread could write the node changing the
         * amount of space available:
         */
-       ret = btree_trans_check_can_insert(trans, stopped_at);
-       if (ret)
-               return ret;
 
-       trans_for_each_update(trans, i) {
-               if (!btree_node_type_needs_gc(i->iter->btree_id))
-                       continue;
+       prefetch(&trans->c->journal.flags);
 
-               if (!fs_usage) {
-                       percpu_down_read(&c->mark_lock);
-                       fs_usage = bch2_fs_usage_scratch_get(c);
-               }
+       trans_for_each_update_sorted(trans, i, iter) {
+               /* Multiple inserts might go to same leaf: */
+               if (!same_leaf_as_prev(trans, iter))
+                       u64s = 0;
 
-               /* Must be called under mark_lock: */
-               if (!bch2_bkey_replicas_marked_locked(c,
-                       bkey_i_to_s_c(i->k), true)) {
-                       ret = BTREE_INSERT_NEED_MARK_REPLICAS;
-                       goto err;
+               u64s += i->k->k.u64s;
+               ret = btree_key_can_insert(trans, i, &u64s);
+               if (ret) {
+                       *stopped_at = i;
+                       return ret;
                }
+
+               if (btree_node_type_needs_gc(i->iter->btree_id))
+                       marking = true;
+       }
+
+       if (marking) {
+               percpu_down_read(&c->mark_lock);
+               fs_usage = bch2_fs_usage_scratch_get(c);
        }
 
        /*
@@ -508,16 +474,20 @@ bch2_trans_commit_write_locked(struct btree_trans *trans,
                                i->k->k.version = MAX_VERSION;
        }
 
+       /* Must be called under mark_lock: */
+       if (marking && trans->fs_usage_deltas &&
+           bch2_replicas_delta_list_apply(c, &fs_usage->u,
+                                          trans->fs_usage_deltas)) {
+               ret = BTREE_INSERT_NEED_MARK_REPLICAS;
+               goto err;
+       }
+
        trans_for_each_update(trans, i)
                if (likely(!(trans->flags & BTREE_INSERT_NOMARK)) &&
                    update_has_nontrans_triggers(i))
                        bch2_mark_update(trans, i, &fs_usage->u, mark_flags);
 
-       if (fs_usage && trans->fs_usage_deltas)
-               bch2_replicas_delta_list_apply(c, &fs_usage->u,
-                                              trans->fs_usage_deltas);
-
-       if (fs_usage)
+       if (marking)
                bch2_trans_fs_usage_apply(trans, fs_usage);
 
        if (unlikely(c->gc_pos.phase))
@@ -526,7 +496,7 @@ bch2_trans_commit_write_locked(struct btree_trans *trans,
        trans_for_each_update(trans, i)
                do_btree_insert_one(trans, i);
 err:
-       if (fs_usage) {
+       if (marking) {
                bch2_fs_usage_scratch_put(c, fs_usage);
                percpu_up_read(&c->mark_lock);
        }
@@ -609,9 +579,17 @@ static inline int do_bch2_trans_commit(struct btree_trans *trans,
         */
        btree_trans_sort_updates(trans);
 
-       btree_trans_lock_write(trans, true);
+       trans_for_each_update_sorted(trans, i, idx)
+               if (!same_leaf_as_prev(trans, idx))
+                       bch2_btree_node_lock_for_insert(trans->c,
+                                               i->iter->l[0].b, i->iter);
+
        ret = bch2_trans_commit_write_locked(trans, stopped_at);
-       btree_trans_lock_write(trans, false);
+
+       trans_for_each_update_sorted(trans, i, idx)
+               if (!same_leaf_as_prev(trans, idx))
+                       bch2_btree_node_unlock_write_inlined(i->iter->l[0].b,
+                                                            i->iter);
 
        /*
         * Drop journal reservation after dropping write locks, since dropping

diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index 0c2ca7601fdea08df826c01c48bc8dc907094ad0..c90c2d1b77063490f9f0ca401f58637035050c30 100644 (file)
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -498,14 +498,18 @@ void bch2_dev_usage_from_buckets(struct bch_fs *c)
        }
 }
 
-static inline void update_replicas(struct bch_fs *c,
-                                  struct bch_fs_usage *fs_usage,
-                                  struct bch_replicas_entry *r,
-                                  s64 sectors)
+static inline int update_replicas(struct bch_fs *c,
+                                 struct bch_fs_usage *fs_usage,
+                                 struct bch_replicas_entry *r,
+                                 s64 sectors)
 {
        int idx = bch2_replicas_entry_idx(c, r);
 
-       BUG_ON(idx < 0);
+       if (idx < 0)
+               return -1;
+
+       if (!fs_usage)
+               return 0;
 
        switch (r->data_type) {
        case BCH_DATA_BTREE:
@@ -519,6 +523,7 @@ static inline void update_replicas(struct bch_fs *c,
                break;
        }
        fs_usage->replicas[idx]         += sectors;
+       return 0;
 }
 
 static inline void update_cached_sectors(struct bch_fs *c,
@@ -579,14 +584,29 @@ static inline void update_cached_sectors_list(struct btree_trans *trans,
        update_replicas_list(trans, &r.e, sectors);
 }
 
-void bch2_replicas_delta_list_apply(struct bch_fs *c,
-                                   struct bch_fs_usage *fs_usage,
-                                   struct replicas_delta_list *r)
+static inline struct replicas_delta *
+replicas_delta_next(struct replicas_delta *d)
+{
+       return (void *) d + replicas_entry_bytes(&d->r) + 8;
+}
+
+int bch2_replicas_delta_list_apply(struct bch_fs *c,
+                                  struct bch_fs_usage *fs_usage,
+                                  struct replicas_delta_list *r)
 {
        struct replicas_delta *d = r->d;
        struct replicas_delta *top = (void *) r->d + r->used;
        unsigned i;
 
+       for (d = r->d; d != top; d = replicas_delta_next(d))
+               if (update_replicas(c, fs_usage, &d->r, d->delta)) {
+                       top = d;
+                       goto unwind;
+               }
+
+       if (!fs_usage)
+               return 0;
+
        fs_usage->nr_inodes += r->nr_inodes;
 
        for (i = 0; i < BCH_REPLICAS_MAX; i++) {
@@ -594,13 +614,11 @@ void bch2_replicas_delta_list_apply(struct bch_fs *c,
                fs_usage->persistent_reserved[i] += r->persistent_reserved[i];
        }
 
-       while (d != top) {
-               BUG_ON((void *) d > (void *) top);
-
-               update_replicas(c, fs_usage, &d->r, d->delta);
-
-               d = (void *) d + replicas_entry_bytes(&d->r) + 8;
-       }
+       return 0;
+unwind:
+       for (d = r->d; d != top; d = replicas_delta_next(d))
+               update_replicas(c, fs_usage, &d->r, -d->delta);
+       return -1;
 }
 
 #define do_mark_fn(fn, c, pos, flags, ...)                             \

diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h
index 5f91a57abc7000ab434a71917d6b2e6724910ab1..336729f763e17b2ef2a685223c848426a182b168 100644 (file)
--- a/fs/bcachefs/buckets.h
+++ b/fs/bcachefs/buckets.h
@@ -279,9 +279,9 @@ int bch2_mark_overwrite(struct btree_trans *, struct btree_iter *,
 int bch2_mark_update(struct btree_trans *, struct btree_insert_entry *,
                     struct bch_fs_usage *, unsigned);
 
-void bch2_replicas_delta_list_apply(struct bch_fs *,
-                                   struct bch_fs_usage *,
-                                   struct replicas_delta_list *);
+int bch2_replicas_delta_list_apply(struct bch_fs *,
+                                  struct bch_fs_usage *,
+                                  struct replicas_delta_list *);
 int bch2_trans_mark_key(struct btree_trans *, struct bkey_s_c,
                        unsigned, s64, unsigned);
 int bch2_trans_mark_update(struct btree_trans *,