bcachefs: btree node prefetching in check_topology

btree_and_journal_iter is old code that we want to get rid of, but we're
not ready to yet.

lack of btree node prefetching is, it turns out, a real performance
issue for fsck on spinning rust, so - add it.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/btree_cache.c b/fs/bcachefs/btree_cache.c
index d7c81beac14afae7ee44f11f28eb424f1b54a063..a8b393bc7567b47e7e7106fd611412a91ecd2c1a 100644 (file)
--- a/fs/bcachefs/btree_cache.c
+++ b/fs/bcachefs/btree_cache.c
@@ -711,6 +711,9 @@ static noinline struct btree *bch2_btree_node_fill(struct btree_trans *trans,
        b = bch2_btree_node_mem_alloc(trans, level != 0);
 
        if (bch2_err_matches(PTR_ERR_OR_ZERO(b), ENOMEM)) {
+               if (!path)
+                       return b;
+
                trans->memory_allocation_failure = true;
                trace_and_count(c, trans_restart_memory_allocation_failure, trans, _THIS_IP_, path);
                return ERR_PTR(btree_trans_restart(trans, BCH_ERR_transaction_restart_fill_mem_alloc_fail));
@@ -760,8 +763,9 @@ static noinline struct btree *bch2_btree_node_fill(struct btree_trans *trans,
        }
 
        if (!six_relock_type(&b->c.lock, lock_type, seq)) {
-               if (path)
-                       trace_and_count(c, trans_restart_relock_after_fill, trans, _THIS_IP_, path);
+               BUG_ON(!path);
+
+               trace_and_count(c, trans_restart_relock_after_fill, trans, _THIS_IP_, path);
                return ERR_PTR(btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_after_fill));
        }
 
@@ -1096,7 +1100,7 @@ int bch2_btree_node_prefetch(struct btree_trans *trans,
        struct btree_cache *bc = &c->btree_cache;
        struct btree *b;
 
-       BUG_ON(trans && !btree_node_locked(path, level + 1));
+       BUG_ON(path && !btree_node_locked(path, level + 1));
        BUG_ON(level >= BTREE_MAX_DEPTH);
 
        b = btree_cache_find(bc, k);

diff --git a/fs/bcachefs/btree_gc.c b/fs/bcachefs/btree_gc.c
index 2caf9f5e0c358603692d0b45302611e655513524..3005b39d3a1a7a6ac262fc54feb3b96debc1683a 100644 (file)
--- a/fs/bcachefs/btree_gc.c
+++ b/fs/bcachefs/btree_gc.c
@@ -390,6 +390,7 @@ again:
        bch2_bkey_buf_init(&prev_k);
        bch2_bkey_buf_init(&cur_k);
        bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b);
+       iter.prefetch = true;
 
        while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
                BUG_ON(bpos_lt(k.k->p, b->data->min_key));
@@ -479,6 +480,7 @@ again:
 
        bch2_btree_and_journal_iter_exit(&iter);
        bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b);
+       iter.prefetch = true;
 
        while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
                bch2_bkey_buf_reassemble(&cur_k, c, k);
@@ -964,6 +966,7 @@ static int bch2_gc_btree_init_recurse(struct btree_trans *trans, struct btree *b
        if (b->c.level > target_depth) {
                bch2_btree_and_journal_iter_exit(&iter);
                bch2_btree_and_journal_iter_init_node_iter(trans, &iter, b);
+               iter.prefetch = true;
 
                while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
                        struct btree *child;

diff --git a/fs/bcachefs/btree_journal_iter.c b/fs/bcachefs/btree_journal_iter.c
index fa907293ba433611942ea6cf4d7f9f58b717aa6c..b7ac93c8fdd8abf0fd3c919dcabcf5d5215b0290 100644 (file)
--- a/fs/bcachefs/btree_journal_iter.c
+++ b/fs/bcachefs/btree_journal_iter.c
@@ -1,7 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0
 
 #include "bcachefs.h"
+#include "bkey_buf.h"
 #include "bset.h"
+#include "btree_cache.h"
 #include "btree_journal_iter.h"
 #include "journal_io.h"
 
@@ -334,9 +336,38 @@ void bch2_btree_and_journal_iter_advance(struct btree_and_journal_iter *iter)
                iter->pos = bpos_successor(iter->pos);
 }
 
+static void btree_and_journal_iter_prefetch(struct btree_and_journal_iter *_iter)
+{
+       struct btree_and_journal_iter iter = *_iter;
+       struct bch_fs *c = iter.trans->c;
+       unsigned level = iter.journal.level;
+       struct bkey_buf tmp;
+       unsigned nr = test_bit(BCH_FS_started, &c->flags)
+               ? (level > 1 ? 0 :  2)
+               : (level > 1 ? 1 : 16);
+
+       iter.prefetch = false;
+       bch2_bkey_buf_init(&tmp);
+
+       while (nr--) {
+               bch2_btree_and_journal_iter_advance(&iter);
+               struct bkey_s_c k = bch2_btree_and_journal_iter_peek(&iter);
+               if (!k.k)
+                       break;
+
+               bch2_bkey_buf_reassemble(&tmp, c, k);
+               bch2_btree_node_prefetch(iter.trans, NULL, tmp.k, iter.journal.btree_id, level - 1);
+       }
+
+       bch2_bkey_buf_exit(&tmp, c);
+}
+
 struct bkey_s_c bch2_btree_and_journal_iter_peek(struct btree_and_journal_iter *iter)
 {
        struct bkey_s_c btree_k, journal_k, ret;
+
+       if (iter->prefetch && iter->journal.level)
+               btree_and_journal_iter_prefetch(iter);
 again:
        if (iter->at_end)
                return bkey_s_c_null;

diff --git a/fs/bcachefs/btree_journal_iter.h b/fs/bcachefs/btree_journal_iter.h
index 1793cf89148b01353175b1e3f1815af4e76bb0f6..c9d19da3ea04803a360a683fa0e01a2838f2433f 100644 (file)
--- a/fs/bcachefs/btree_journal_iter.h
+++ b/fs/bcachefs/btree_journal_iter.h
@@ -23,6 +23,7 @@ struct btree_and_journal_iter {
        struct journal_iter     journal;
        struct bpos             pos;
        bool                    at_end;
+       bool                    prefetch;
 };
 
 struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *, enum btree_id,