From: Kent Overstreet Date: Fri, 12 Feb 2021 02:57:32 +0000 (-0500) Subject: bcachefs: iter->real_pos X-Git-Url: http://git.maquefel.me/?a=commitdiff_plain;h=7e1a3aa9dfcb9cd8f46085df86f158a1f23085dc;p=linux.git bcachefs: iter->real_pos We need to differentiate between the search position of a btree iterator, vs. what it actually points at (what we found). This matters for extents, where iter->pos will typically be the start of the key we found and iter->real_pos will be the end of the key we found (which soon won't necessarily be in the same btree node!) and it will also matter for snapshots. Signed-off-by: Kent Overstreet Signed-off-by: Kent Overstreet --- diff --git a/fs/bcachefs/btree_iter.c b/fs/bcachefs/btree_iter.c index 4012a2c0f0080..531732e309508 100644 --- a/fs/bcachefs/btree_iter.c +++ b/fs/bcachefs/btree_iter.c @@ -34,13 +34,13 @@ static inline struct bpos btree_iter_search_key(struct btree_iter *iter) static inline bool btree_iter_pos_before_node(struct btree_iter *iter, struct btree *b) { - return bkey_cmp(btree_iter_search_key(iter), b->data->min_key) < 0; + return bkey_cmp(iter->real_pos, b->data->min_key) < 0; } static inline bool btree_iter_pos_after_node(struct btree_iter *iter, struct btree *b) { - return bkey_cmp(b->key.k.p, btree_iter_search_key(iter)) < 0; + return bkey_cmp(b->key.k.p, iter->real_pos) < 0; } static inline bool btree_iter_pos_in_node(struct btree_iter *iter, @@ -491,7 +491,6 @@ static void bch2_btree_iter_verify_cached(struct btree_iter *iter) static void bch2_btree_iter_verify_level(struct btree_iter *iter, unsigned level) { - struct bpos pos = btree_iter_search_key(iter); struct btree_iter_level *l = &iter->l[level]; struct btree_node_iter tmp = l->iter; bool locked = btree_node_locked(iter, level); @@ -539,12 +538,12 @@ static void bch2_btree_iter_verify_level(struct btree_iter *iter, : bch2_btree_node_iter_prev_all(&tmp, l->b); k = bch2_btree_node_iter_peek_all(&l->iter, l->b); - if (p && bkey_iter_pos_cmp(l->b, p, &pos) >= 0) { + if (p && bkey_iter_pos_cmp(l->b, p, &iter->real_pos) >= 0) { msg = "before"; goto err; } - if (k && bkey_iter_pos_cmp(l->b, k, &pos) < 0) { + if (k && bkey_iter_pos_cmp(l->b, k, &iter->real_pos) < 0) { msg = "after"; goto err; } @@ -567,12 +566,11 @@ err: } panic("iterator should be %s key at level %u:\n" - "iter pos %s %llu:%llu\n" + "iter pos %llu:%llu\n" "prev key %s\n" "cur key %s\n", msg, level, - iter->flags & BTREE_ITER_IS_EXTENTS ? ">" : "=>", - iter->pos.inode, iter->pos.offset, + iter->real_pos.inode, iter->real_pos.offset, buf1, buf2); } @@ -580,12 +578,24 @@ static void bch2_btree_iter_verify(struct btree_iter *iter) { unsigned i; - bch2_btree_trans_verify_locks(iter->trans); + EBUG_ON(iter->btree_id >= BTREE_ID_NR); + + bch2_btree_iter_verify_locks(iter); for (i = 0; i < BTREE_MAX_DEPTH; i++) bch2_btree_iter_verify_level(iter, i); } +static void bch2_btree_iter_verify_entry_exit(struct btree_iter *iter) +{ + enum btree_iter_type type = btree_iter_type(iter); + + BUG_ON((type == BTREE_ITER_KEYS || + type == BTREE_ITER_CACHED) && + (bkey_cmp(iter->pos, bkey_start_pos(&iter->k)) < 0 || + bkey_cmp(iter->pos, iter->k.p) > 0)); +} + void bch2_btree_trans_verify_iters(struct btree_trans *trans, struct btree *b) { struct btree_iter *iter; @@ -601,6 +611,7 @@ void bch2_btree_trans_verify_iters(struct btree_trans *trans, struct btree *b) static inline void bch2_btree_iter_verify_level(struct btree_iter *iter, unsigned l) {} static inline void bch2_btree_iter_verify(struct btree_iter *iter) {} +static inline void bch2_btree_iter_verify_entry_exit(struct btree_iter *iter) {} #endif @@ -626,12 +637,11 @@ static void __bch2_btree_iter_fix_key_modified(struct btree_iter *iter, struct bkey_packed *where) { struct btree_iter_level *l = &iter->l[b->c.level]; - struct bpos pos = btree_iter_search_key(iter); if (where != bch2_btree_node_iter_peek_all(&l->iter, l->b)) return; - if (bkey_iter_pos_cmp(l->b, where, &pos) < 0) + if (bkey_iter_pos_cmp(l->b, where, &iter->real_pos) < 0) bch2_btree_node_iter_advance(&l->iter, l->b); btree_iter_set_dirty(iter, BTREE_ITER_NEED_PEEK); @@ -666,7 +676,6 @@ static void __bch2_btree_node_iter_fix(struct btree_iter *iter, bool iter_current_key_modified = orig_iter_pos >= offset && orig_iter_pos <= offset + clobber_u64s; - struct bpos iter_pos = btree_iter_search_key(iter); btree_node_iter_for_each(node_iter, set) if (set->end == old_end) @@ -674,7 +683,7 @@ static void __bch2_btree_node_iter_fix(struct btree_iter *iter, /* didn't find the bset in the iterator - might have to readd it: */ if (new_u64s && - bkey_iter_pos_cmp(b, where, &iter_pos) >= 0) { + bkey_iter_pos_cmp(b, where, &iter->real_pos) >= 0) { bch2_btree_node_iter_push(node_iter, b, where, end); goto fixup_done; } else { @@ -689,7 +698,7 @@ found: return; if (new_u64s && - bkey_iter_pos_cmp(b, where, &iter_pos) >= 0) { + bkey_iter_pos_cmp(b, where, &iter->real_pos) >= 0) { set->k = offset; } else if (set->k < offset + clobber_u64s) { set->k = offset + new_u64s; @@ -825,12 +834,11 @@ static inline bool btree_iter_advance_to_pos(struct btree_iter *iter, struct btree_iter_level *l, int max_advance) { - struct bpos pos = btree_iter_search_key(iter); struct bkey_packed *k; int nr_advanced = 0; while ((k = bch2_btree_node_iter_peek_all(&l->iter, l->b)) && - bkey_iter_pos_cmp(l->b, k, &pos) < 0) { + bkey_iter_pos_cmp(l->b, k, &iter->real_pos) < 0) { if (max_advance > 0 && nr_advanced >= max_advance) return false; @@ -893,10 +901,9 @@ static void btree_iter_verify_new_node(struct btree_iter *iter, struct btree *b) static inline void __btree_iter_init(struct btree_iter *iter, unsigned level) { - struct bpos pos = btree_iter_search_key(iter); struct btree_iter_level *l = &iter->l[level]; - bch2_btree_node_iter_init(&l->iter, l->b, &pos); + bch2_btree_node_iter_init(&l->iter, l->b, &iter->real_pos); /* * Iterators to interior nodes should always be pointed at the first non @@ -1342,21 +1349,6 @@ int __must_check __bch2_btree_iter_traverse(struct btree_iter *iter) return ret; } -static inline void bch2_btree_iter_checks(struct btree_iter *iter) -{ - enum btree_iter_type type = btree_iter_type(iter); - - EBUG_ON(iter->btree_id >= BTREE_ID_NR); - - BUG_ON((type == BTREE_ITER_KEYS || - type == BTREE_ITER_CACHED) && - (bkey_cmp(iter->pos, bkey_start_pos(&iter->k)) < 0 || - bkey_cmp(iter->pos, iter->k.p) > 0)); - - bch2_btree_iter_verify_locks(iter); - bch2_btree_iter_verify_level(iter, iter->level); -} - /* Iterate across nodes (leaf and interior nodes) */ struct btree *bch2_btree_iter_peek_node(struct btree_iter *iter) @@ -1365,7 +1357,7 @@ struct btree *bch2_btree_iter_peek_node(struct btree_iter *iter) int ret; EBUG_ON(btree_iter_type(iter) != BTREE_ITER_NODES); - bch2_btree_iter_checks(iter); + bch2_btree_iter_verify(iter); if (iter->uptodate == BTREE_ITER_UPTODATE) return iter->l[iter->level].b; @@ -1380,7 +1372,7 @@ struct btree *bch2_btree_iter_peek_node(struct btree_iter *iter) BUG_ON(bkey_cmp(b->key.k.p, iter->pos) < 0); - iter->pos = b->key.k.p; + iter->pos = iter->real_pos = b->key.k.p; iter->uptodate = BTREE_ITER_UPTODATE; bch2_btree_iter_verify(iter); @@ -1394,7 +1386,7 @@ struct btree *bch2_btree_iter_next_node(struct btree_iter *iter) int ret; EBUG_ON(btree_iter_type(iter) != BTREE_ITER_NODES); - bch2_btree_iter_checks(iter); + bch2_btree_iter_verify(iter); /* already got to end? */ if (!btree_iter_node(iter, iter->level)) @@ -1431,7 +1423,7 @@ struct btree *bch2_btree_iter_next_node(struct btree_iter *iter) if (btree_node_read_locked(iter, iter->level)) btree_node_unlock(iter, iter->level); - iter->pos = bkey_successor(iter->pos); + iter->pos = iter->real_pos = bkey_successor(iter->pos); iter->level = iter->min_depth; btree_iter_set_dirty(iter, BTREE_ITER_NEED_TRAVERSE); @@ -1442,7 +1434,7 @@ struct btree *bch2_btree_iter_next_node(struct btree_iter *iter) b = iter->l[iter->level].b; } - iter->pos = b->key.k.p; + iter->pos = iter->real_pos = b->key.k.p; iter->uptodate = BTREE_ITER_UPTODATE; bch2_btree_iter_verify(iter); @@ -1490,57 +1482,55 @@ out: btree_iter_set_dirty(iter, BTREE_ITER_NEED_PEEK); } -void __bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos new_pos, - bool strictly_greater) +static void btree_iter_set_search_pos(struct btree_iter *iter, struct bpos new_pos) { - struct bpos old = btree_iter_search_key(iter); - int cmp; + int cmp = bkey_cmp(new_pos, iter->real_pos); - iter->flags &= ~BTREE_ITER_IS_EXTENTS; - iter->flags |= strictly_greater ? BTREE_ITER_IS_EXTENTS : 0; + iter->real_pos = new_pos; + + btree_iter_pos_changed(iter, cmp); + bch2_btree_iter_verify(iter); +} + +void __bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos new_pos, + bool strictly_greater) +{ bkey_init(&iter->k); iter->k.p = iter->pos = new_pos; - cmp = bkey_cmp(btree_iter_search_key(iter), old); + iter->flags &= ~BTREE_ITER_IS_EXTENTS; + iter->flags |= strictly_greater ? BTREE_ITER_IS_EXTENTS : 0; - btree_iter_pos_changed(iter, cmp); + btree_iter_set_search_pos(iter, btree_iter_search_key(iter)); } void bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos new_pos) { - int cmp = bkey_cmp(new_pos, iter->pos); - - bkey_init(&iter->k); - iter->k.p = iter->pos = new_pos; - - btree_iter_pos_changed(iter, cmp); + __bch2_btree_iter_set_pos(iter, new_pos, + (iter->flags & BTREE_ITER_IS_EXTENTS) != 0); } static inline bool bch2_btree_iter_advance_pos(struct btree_iter *iter) { struct bpos pos = iter->k.p; + bool ret = bkey_cmp(pos, POS_MAX) != 0; - if (unlikely(!bkey_cmp(pos, POS_MAX))) - return false; - - if (!(iter->flags & BTREE_ITER_IS_EXTENTS)) + if (ret && !(iter->flags & BTREE_ITER_IS_EXTENTS)) pos = bkey_successor(pos); bch2_btree_iter_set_pos(iter, pos); - return true; + return ret; } static inline bool bch2_btree_iter_rewind_pos(struct btree_iter *iter) { struct bpos pos = bkey_start_pos(&iter->k); + bool ret = bkey_cmp(pos, POS_MIN) != 0; - if (unlikely(!bkey_cmp(pos, POS_MIN))) - return false; - - if (!(iter->flags & BTREE_ITER_IS_EXTENTS)) + if (ret && !(iter->flags & BTREE_ITER_IS_EXTENTS)) pos = bkey_predecessor(pos); bch2_btree_iter_set_pos(iter, pos); - return true; + return ret; } static inline bool btree_iter_set_pos_to_next_leaf(struct btree_iter *iter) @@ -1548,10 +1538,16 @@ static inline bool btree_iter_set_pos_to_next_leaf(struct btree_iter *iter) struct bpos next_pos = iter->l[0].b->key.k.p; bool ret = bkey_cmp(next_pos, POS_MAX) != 0; - if (ret && !(iter->flags & BTREE_ITER_IS_EXTENTS)) - next_pos = bkey_successor(next_pos); + /* + * Typically, we don't want to modify iter->pos here, since that + * indicates where we searched from - unless we got to the end of the + * btree, in that case we want iter->pos to reflect that: + */ + if (ret) + btree_iter_set_search_pos(iter, bkey_successor(next_pos)); + else + bch2_btree_iter_set_pos(iter, POS_MAX); - bch2_btree_iter_set_pos(iter, next_pos); return ret; } @@ -1560,14 +1556,11 @@ static inline bool btree_iter_set_pos_to_prev_leaf(struct btree_iter *iter) struct bpos next_pos = iter->l[0].b->data->min_key; bool ret = bkey_cmp(next_pos, POS_MIN) != 0; - if (ret) { - next_pos = bkey_predecessor(next_pos); - - if (iter->flags & BTREE_ITER_IS_EXTENTS) - next_pos = bkey_predecessor(next_pos); - } + if (ret) + btree_iter_set_search_pos(iter, bkey_predecessor(next_pos)); + else + bch2_btree_iter_set_pos(iter, POS_MIN); - bch2_btree_iter_set_pos(iter, next_pos); return ret; } @@ -1610,7 +1603,10 @@ struct bkey_s_c bch2_btree_iter_peek(struct btree_iter *iter) int ret; EBUG_ON(btree_iter_type(iter) != BTREE_ITER_KEYS); - bch2_btree_iter_checks(iter); + bch2_btree_iter_verify(iter); + bch2_btree_iter_verify_entry_exit(iter); + + btree_iter_set_search_pos(iter, btree_iter_search_key(iter)); if (iter->uptodate == BTREE_ITER_UPTODATE && !bkey_deleted(&iter->k)) @@ -1636,9 +1632,12 @@ struct bkey_s_c bch2_btree_iter_peek(struct btree_iter *iter) if (bkey_cmp(bkey_start_pos(k.k), iter->pos) > 0) iter->pos = bkey_start_pos(k.k); + iter->real_pos = k.k->p; + iter->uptodate = BTREE_ITER_UPTODATE; - bch2_btree_iter_verify_level(iter, 0); + bch2_btree_iter_verify_entry_exit(iter); + bch2_btree_iter_verify(iter); return k; } @@ -1692,7 +1691,7 @@ struct bkey_s_c bch2_btree_iter_peek_with_updates(struct btree_iter *iter) int ret; EBUG_ON(btree_iter_type(iter) != BTREE_ITER_KEYS); - bch2_btree_iter_checks(iter); + bch2_btree_iter_verify(iter); while (1) { ret = bch2_btree_iter_traverse(iter); @@ -1714,8 +1713,8 @@ struct bkey_s_c bch2_btree_iter_peek_with_updates(struct btree_iter *iter) } /* - * iter->pos should always be equal to the key we just - * returned - except extents can straddle iter->pos: + * iter->pos should be mononotically increasing, and always be equal to + * the key we just returned - except extents can straddle iter->pos: */ if (bkey_cmp(bkey_start_pos(k.k), iter->pos) > 0) iter->pos = bkey_start_pos(k.k); @@ -1744,7 +1743,10 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) int ret; EBUG_ON(btree_iter_type(iter) != BTREE_ITER_KEYS); - bch2_btree_iter_checks(iter); + bch2_btree_iter_verify(iter); + bch2_btree_iter_verify_entry_exit(iter); + + btree_iter_set_search_pos(iter, iter->pos); if (iter->uptodate == BTREE_ITER_UPTODATE && !bkey_deleted(&iter->k)) @@ -1752,8 +1754,10 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) while (1) { ret = bch2_btree_iter_traverse(iter); - if (unlikely(ret)) - return bkey_s_c_err(ret); + if (unlikely(ret)) { + k = bkey_s_c_err(ret); + goto no_key; + } k = __btree_iter_peek(iter, l); if (!k.k || @@ -1765,8 +1769,10 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) if (likely(k.k)) break; - if (!btree_iter_set_pos_to_prev_leaf(iter)) - return bkey_s_c_null; + if (!btree_iter_set_pos_to_prev_leaf(iter)) { + k = bkey_s_c_null; + goto no_key; + } } EBUG_ON(bkey_cmp(bkey_start_pos(k.k), pos) > 0); @@ -1774,10 +1780,23 @@ struct bkey_s_c bch2_btree_iter_peek_prev(struct btree_iter *iter) /* Extents can straddle iter->pos: */ if (bkey_cmp(k.k->p, pos) < 0) iter->pos = k.k->p; - iter->uptodate = BTREE_ITER_UPTODATE; - - bch2_btree_iter_verify_level(iter, 0); + iter->real_pos = k.k->p; + iter->uptodate = BTREE_ITER_UPTODATE; +out: + bch2_btree_iter_verify_entry_exit(iter); + bch2_btree_iter_verify(iter); return k; +no_key: + /* + * __btree_iter_peek() may have set iter->k to a key we didn't want, and + * then we errored going to the previous leaf - make sure it's + * consistent with iter->pos: + */ + BUG_ON(bkey_cmp(pos, iter->pos) && + bkey_cmp(iter->pos, POS_MIN)); + bkey_init(&iter->k); + iter->k.p = iter->pos; + goto out; } /** @@ -1830,7 +1849,7 @@ __bch2_btree_iter_peek_slot_extents(struct btree_iter *iter) */ EBUG_ON(bkey_cmp(k.k->p, iter->pos) <= 0); - bch2_btree_iter_verify_level(iter, 0); + bch2_btree_iter_verify(iter); return k; } @@ -1853,7 +1872,9 @@ __bch2_btree_iter_peek_slot_extents(struct btree_iter *iter) iter->k = n; iter->uptodate = BTREE_ITER_UPTODATE; - bch2_btree_iter_verify_level(iter, 0); + bch2_btree_iter_verify_entry_exit(iter); + bch2_btree_iter_verify(iter); + return (struct bkey_s_c) { &iter->k, NULL }; } @@ -1864,7 +1885,10 @@ struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) int ret; EBUG_ON(btree_iter_type(iter) != BTREE_ITER_KEYS); - bch2_btree_iter_checks(iter); + bch2_btree_iter_verify(iter); + bch2_btree_iter_verify_entry_exit(iter); + + btree_iter_set_search_pos(iter, btree_iter_search_key(iter)); if (iter->uptodate == BTREE_ITER_UPTODATE) return btree_iter_peek_uptodate(iter); @@ -1888,7 +1912,8 @@ struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_iter *iter) } iter->uptodate = BTREE_ITER_UPTODATE; - bch2_btree_iter_verify_level(iter, 0); + bch2_btree_iter_verify_entry_exit(iter); + bch2_btree_iter_verify(iter); return k; } @@ -1906,7 +1931,7 @@ struct bkey_s_c bch2_btree_iter_peek_cached(struct btree_iter *iter) int ret; EBUG_ON(btree_iter_type(iter) != BTREE_ITER_CACHED); - bch2_btree_iter_checks(iter); + bch2_btree_iter_verify(iter); ret = bch2_btree_iter_traverse(iter); if (unlikely(ret)) @@ -1937,6 +1962,7 @@ static inline void bch2_btree_iter_init(struct btree_trans *trans, bkey_init(&iter->k); iter->k.p = pos; iter->flags = flags; + iter->real_pos = btree_iter_search_key(iter); iter->uptodate = BTREE_ITER_NEED_TRAVERSE; iter->btree_id = btree_id; iter->level = 0; @@ -2076,7 +2102,7 @@ static struct btree_iter *__btree_trans_get_iter(struct btree_trans *trans, if (best && bkey_cmp(bpos_diff(best->pos, pos), - bpos_diff(iter->pos, pos)) < 0) + bpos_diff(iter->real_pos, pos)) < 0) continue; best = iter; diff --git a/fs/bcachefs/btree_types.h b/fs/bcachefs/btree_types.h index e51e3c7868de0..80bb31a533395 100644 --- a/fs/bcachefs/btree_types.h +++ b/fs/bcachefs/btree_types.h @@ -247,6 +247,8 @@ enum btree_iter_uptodate { struct btree_iter { struct btree_trans *trans; struct bpos pos; + /* what we're searching for/what the iterator actually points to: */ + struct bpos real_pos; struct bpos pos_after_commit; u16 flags; diff --git a/fs/bcachefs/btree_update_leaf.c b/fs/bcachefs/btree_update_leaf.c index 2f94b8917a767..5e0ce7cde0178 100644 --- a/fs/bcachefs/btree_update_leaf.c +++ b/fs/bcachefs/btree_update_leaf.c @@ -219,7 +219,7 @@ static inline void btree_insert_entry_checks(struct btree_trans *trans, { struct bch_fs *c = trans->c; - BUG_ON(bkey_cmp(insert->k.p, iter->pos)); + BUG_ON(bkey_cmp(insert->k.p, iter->real_pos)); BUG_ON(bch2_debug_check_bkeys && bch2_bkey_invalid(c, bkey_i_to_s_c(insert), __btree_node_type(iter->level, iter->btree_id)));