bcachefs: Fix journal_keys_search() overhead

Previously, on every btree_iter_peek() operation we were searching the
journal keys, doing a full binary search - which was slow.

This patch fixes that by saving our position in the journal keys, so
that we only do a full binary search when moving our position backwards
or a large jump forwards.

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>

diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c
index b840035dca55f8394814ecafc2a7a4ce353abe35..3ce0571651b5ec880384fed4cb970d3ee64b3afc 100644 (file)
--- a/fs/bcachefs/btree_iter.c
+++ b/fs/bcachefs/btree_iter.c
@@ -2242,13 +2242,30 @@ static inline struct bkey_i *btree_trans_peek_updates(struct btree_iter *iter)
                : NULL;
 }
 
+struct bkey_i *bch2_btree_journal_peek(struct btree_trans *trans,
+                                      struct btree_iter *iter,
+                                      struct bpos end_pos)
+{
+       struct bkey_i *k;
+
+       if (bpos_cmp(iter->path->pos, iter->journal_pos) < 0)
+               iter->journal_idx = 0;
+
+       k = bch2_journal_keys_peek_upto(trans->c, iter->btree_id,
+                                       iter->path->level,
+                                       iter->path->pos,
+                                       end_pos,
+                                       &iter->journal_idx);
+
+       iter->journal_pos = k ? k->k.p : end_pos;
+       return k;
+}
+
 static noinline
 struct bkey_s_c btree_trans_peek_slot_journal(struct btree_trans *trans,
                                              struct btree_iter *iter)
 {
-       struct bkey_i *k = bch2_journal_keys_peek_slot(trans->c, iter->btree_id,
-                                                      iter->path->level,
-                                                      iter->path->pos);
+       struct bkey_i *k = bch2_btree_journal_peek(trans, iter, iter->path->pos);
 
        if (k) {
                iter->k = k->k;
@@ -2264,8 +2281,7 @@ struct bkey_s_c btree_trans_peek_journal(struct btree_trans *trans,
                                         struct bkey_s_c k)
 {
        struct bkey_i *next_journal =
-               bch2_journal_keys_peek_upto(trans->c, iter->btree_id, 0,
-                               iter->path->pos,
+               bch2_btree_journal_peek(trans, iter,
                                k.k ? k.k->p : iter->path->l[0].b->key.k.p);
 
        if (next_journal) {
@@ -3072,6 +3088,8 @@ static void __bch2_trans_iter_init(struct btree_trans *trans,
        iter->k.type    = KEY_TYPE_deleted;
        iter->k.p       = pos;
        iter->k.size    = 0;
+       iter->journal_idx = 0;
+       iter->journal_pos = POS_MIN;
 
        iter->path = bch2_path_get(trans, btree_id, iter->pos,
                                   locks_want, depth, flags);

diff --git a/fs/bcachefs/btree_iter.h b/fs/bcachefs/btree_iter.h
index dc6f07492bc9a8711f5e69bd41a819dd70aefc74..83587383a41ff822a4e937ca418e22340e9faf28 100644 (file)
--- a/fs/bcachefs/btree_iter.h
+++ b/fs/bcachefs/btree_iter.h
@@ -175,6 +175,9 @@ struct btree_path *bch2_path_get(struct btree_trans *, enum btree_id, struct bpo
                                 unsigned, unsigned, unsigned);
 inline struct bkey_s_c bch2_btree_path_peek_slot(struct btree_path *, struct bkey *);
 
+struct bkey_i *bch2_btree_journal_peek_slot(struct btree_trans *,
+                                       struct btree_iter *, struct bpos);
+
 #ifdef CONFIG_BCACHEFS_DEBUG
 void bch2_trans_verify_paths(struct btree_trans *);
 void bch2_trans_verify_locks(struct btree_trans *);

diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h
index 4f359ff79334ffbccb03b7bb3a253fb6b64a8622..82c8c148c4bc25e7e555e152d69421b3be740ef6 100644 (file)
--- a/fs/bcachefs/btree_types.h
+++ b/fs/bcachefs/btree_types.h
@@ -292,6 +292,10 @@ struct btree_iter {
         * bch2_btree_iter_next_slot() can correctly advance pos.
         */
        struct bkey             k;
+
+       /* BTREE_ITER_WITH_JOURNAL: */
+       size_t                  journal_idx;
+       struct bpos             journal_pos;
 };
 
 struct btree_key_cache {

diff --git a/fs/bcachefs/recovery.c b/fs/bcachefs/recovery.c
index 2e782d5d968e6126564f0ac3eccd9dde3f5ca0d2..edb04f65a148c16b65baacf4b278f6291e825e49 100644 (file)
--- a/fs/bcachefs/recovery.c
+++ b/fs/bcachefs/recovery.c
@@ -86,9 +86,9 @@ static inline struct journal_key *idx_to_key(struct journal_keys *keys, size_t i
        return keys->d + idx_to_pos(keys, idx);
 }
 
-size_t bch2_journal_key_search(struct journal_keys *keys,
-                              enum btree_id id, unsigned level,
-                              struct bpos pos)
+static size_t __bch2_journal_key_search(struct journal_keys *keys,
+                                       enum btree_id id, unsigned level,
+                                       struct bpos pos)
 {
        size_t l = 0, r = keys->nr, m;
 
@@ -106,26 +106,42 @@ size_t bch2_journal_key_search(struct journal_keys *keys,
        BUG_ON(l &&
               __journal_key_cmp(id, level, pos, idx_to_key(keys, l - 1)) <= 0);
 
-       return idx_to_pos(keys, l);
+       return l;
+}
+
+static size_t bch2_journal_key_search(struct journal_keys *keys,
+                                     enum btree_id id, unsigned level,
+                                     struct bpos pos)
+{
+       return idx_to_pos(keys, __bch2_journal_key_search(keys, id, level, pos));
 }
 
 struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *c, enum btree_id btree_id,
                                           unsigned level, struct bpos pos,
-                                          struct bpos end_pos)
+                                          struct bpos end_pos, size_t *idx)
 {
        struct journal_keys *keys = &c->journal_keys;
-       size_t idx = bch2_journal_key_search(keys, btree_id, level, pos);
-
-       while (idx < keys->size &&
-              keys->d[idx].btree_id == btree_id &&
-              keys->d[idx].level == level &&
-              bpos_cmp(keys->d[idx].k->k.p, end_pos) <= 0) {
-               if (!keys->d[idx].overwritten)
-                       return keys->d[idx].k;
-
-               idx++;
-               if (idx == keys->gap)
-                       idx += keys->size - keys->nr;
+       unsigned iters = 0;
+       struct journal_key *k;
+search:
+       if (!*idx)
+               *idx = __bch2_journal_key_search(keys, btree_id, level, pos);
+
+       while (*idx < keys->nr &&
+              (k = idx_to_key(keys, *idx),
+               k->btree_id == btree_id &&
+               k->level == level &&
+               bpos_cmp(k->k->k.p, end_pos) <= 0)) {
+               if (bpos_cmp(k->k->k.p, pos) >= 0 &&
+                   !k->overwritten)
+                       return k->k;
+
+               (*idx)++;
+               iters++;
+               if (iters == 10) {
+                       *idx = 0;
+                       goto search;
+               }
        }
 
        return NULL;
@@ -134,7 +150,9 @@ struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *c, enum btree_id btree
 struct bkey_i *bch2_journal_keys_peek_slot(struct bch_fs *c, enum btree_id btree_id,
                                           unsigned level, struct bpos pos)
 {
-       return bch2_journal_keys_peek_upto(c, btree_id, level, pos, pos);
+       size_t idx = 0;
+
+       return bch2_journal_keys_peek_upto(c, btree_id, level, pos, pos, &idx);
 }
 
 static void journal_iters_fix(struct bch_fs *c)

diff --git a/fs/bcachefs/recovery.h b/fs/bcachefs/recovery.h
index e05aac64185d327e3aef1e4bc2aff2f9dae2ed30..52db06b29310f5f9449914bab965d2d2427b959d 100644 (file)
--- a/fs/bcachefs/recovery.h
+++ b/fs/bcachefs/recovery.h
@@ -28,10 +28,8 @@ struct btree_and_journal_iter {
        }                       last;
 };
 
-size_t bch2_journal_key_search(struct journal_keys *, enum btree_id,
-                              unsigned, struct bpos);
 struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *, enum btree_id,
-                                          unsigned, struct bpos, struct bpos);
+                               unsigned, struct bpos, struct bpos, size_t *);
 struct bkey_i *bch2_journal_keys_peek_slot(struct bch_fs *, enum btree_id,
                                           unsigned, struct bpos);