static inline bool btree_iter_pos_before_node(struct btree_iter *iter,
struct btree *b)
{
- return bkey_cmp(btree_iter_search_key(iter), b->data->min_key) < 0;
+ return bkey_cmp(iter->real_pos, b->data->min_key) < 0;
}
static inline bool btree_iter_pos_after_node(struct btree_iter *iter,
struct btree *b)
{
- return bkey_cmp(b->key.k.p, btree_iter_search_key(iter)) < 0;
+ return bkey_cmp(b->key.k.p, iter->real_pos) < 0;
}
static inline bool btree_iter_pos_in_node(struct btree_iter *iter,
static void bch2_btree_iter_verify_level(struct btree_iter *iter,
unsigned level)
{
- struct bpos pos = btree_iter_search_key(iter);
struct btree_iter_level *l = &iter->l[level];
struct btree_node_iter tmp = l->iter;
bool locked = btree_node_locked(iter, level);
: bch2_btree_node_iter_prev_all(&tmp, l->b);
k = bch2_btree_node_iter_peek_all(&l->iter, l->b);
- if (p && bkey_iter_pos_cmp(l->b, p, &pos) >= 0) {
+ if (p && bkey_iter_pos_cmp(l->b, p, &iter->real_pos) >= 0) {
msg = "before";
goto err;
}
- if (k && bkey_iter_pos_cmp(l->b, k, &pos) < 0) {
+ if (k && bkey_iter_pos_cmp(l->b, k, &iter->real_pos) < 0) {
msg = "after";
goto err;
}
}
panic("iterator should be %s key at level %u:\n"
- "iter pos %s %llu:%llu\n"
+ "iter pos %llu:%llu\n"
"prev key %s\n"
"cur key %s\n",
msg, level,
- iter->flags & BTREE_ITER_IS_EXTENTS ? ">" : "=>",
- iter->pos.inode, iter->pos.offset,
+ iter->real_pos.inode, iter->real_pos.offset,
buf1, buf2);
}
{
unsigned i;
- bch2_btree_trans_verify_locks(iter->trans);
+ EBUG_ON(iter->btree_id >= BTREE_ID_NR);
+
+ bch2_btree_iter_verify_locks(iter);
for (i = 0; i < BTREE_MAX_DEPTH; i++)
bch2_btree_iter_verify_level(iter, i);
}
+static void bch2_btree_iter_verify_entry_exit(struct btree_iter *iter)
+{
+ enum btree_iter_type type = btree_iter_type(iter);
+
+ BUG_ON((type == BTREE_ITER_KEYS ||
+ type == BTREE_ITER_CACHED) &&
+ (bkey_cmp(iter->pos, bkey_start_pos(&iter->k)) < 0 ||
+ bkey_cmp(iter->pos, iter->k.p) > 0));
+}
+
void bch2_btree_trans_verify_iters(struct btree_trans *trans, struct btree *b)
{
struct btree_iter *iter;
static inline void bch2_btree_iter_verify_level(struct btree_iter *iter, unsigned l) {}
static inline void bch2_btree_iter_verify(struct btree_iter *iter) {}
+static inline void bch2_btree_iter_verify_entry_exit(struct btree_iter *iter) {}
#endif
struct bkey_packed *where)
{
struct btree_iter_level *l = &iter->l[b->c.level];
- struct bpos pos = btree_iter_search_key(iter);
if (where != bch2_btree_node_iter_peek_all(&l->iter, l->b))
return;
- if (bkey_iter_pos_cmp(l->b, where, &pos) < 0)
+ if (bkey_iter_pos_cmp(l->b, where, &iter->real_pos) < 0)
bch2_btree_node_iter_advance(&l->iter, l->b);
btree_iter_set_dirty(iter, BTREE_ITER_NEED_PEEK);
bool iter_current_key_modified =
orig_iter_pos >= offset &&
orig_iter_pos <= offset + clobber_u64s;
- struct bpos iter_pos = btree_iter_search_key(iter);
btree_node_iter_for_each(node_iter, set)
if (set->end == old_end)
/* didn't find the bset in the iterator - might have to readd it: */
if (new_u64s &&
- bkey_iter_pos_cmp(b, where, &iter_pos) >= 0) {
+ bkey_iter_pos_cmp(b, where, &iter->real_pos) >= 0) {
bch2_btree_node_iter_push(node_iter, b, where, end);
goto fixup_done;
} else {
return;
if (new_u64s &&
- bkey_iter_pos_cmp(b, where, &iter_pos) >= 0) {
+ bkey_iter_pos_cmp(b, where, &iter->real_pos) >= 0) {
set->k = offset;
} else if (set->k < offset + clobber_u64s) {
set->k = offset + new_u64s;
struct btree_iter_level *l,
int max_advance)
{
- struct bpos pos = btree_iter_search_key(iter);
struct bkey_packed *k;
int nr_advanced = 0;
while ((k = bch2_btree_node_iter_peek_all(&l->iter, l->b)) &&
- bkey_iter_pos_cmp(l->b, k, &pos) < 0) {
+ bkey_iter_pos_cmp(l->b, k, &iter->real_pos) < 0) {
if (max_advance > 0 && nr_advanced >= max_advance)
return false;
static inline void __btree_iter_init(struct btree_iter *iter,
unsigned level)
{
- struct bpos pos = btree_iter_search_key(iter);
struct btree_iter_level *l = &iter->l[level];
- bch2_btree_node_iter_init(&l->iter, l->b, &pos);
+ bch2_btree_node_iter_init(&l->iter, l->b, &iter->real_pos);
/*
* Iterators to interior nodes should always be pointed at the first non
return ret;
}
-static inline void bch2_btree_iter_checks(struct btree_iter *iter)
-{
- enum btree_iter_type type = btree_iter_type(iter);
-
- EBUG_ON(iter->btree_id >= BTREE_ID_NR);
-
- BUG_ON((type == BTREE_ITER_KEYS ||
- type == BTREE_ITER_CACHED) &&
- (bkey_cmp(iter->pos, bkey_start_pos(&iter->k)) < 0 ||
- bkey_cmp(iter->pos, iter->k.p) > 0));
-
- bch2_btree_iter_verify_locks(iter);
- bch2_btree_iter_verify_level(iter, iter->level);
-}
-
/* Iterate across nodes (leaf and interior nodes) */
struct btree *bch2_btree_iter_peek_node(struct btree_iter *iter)
int ret;
EBUG_ON(btree_iter_type(iter) != BTREE_ITER_NODES);
- bch2_btree_iter_checks(iter);
+ bch2_btree_iter_verify(iter);
if (iter->uptodate == BTREE_ITER_UPTODATE)
return iter->l[iter->level].b;
BUG_ON(bkey_cmp(b->key.k.p, iter->pos) < 0);
- iter->pos = b->key.k.p;
+ iter->pos = iter->real_pos = b->key.k.p;
iter->uptodate = BTREE_ITER_UPTODATE;
bch2_btree_iter_verify(iter);
int ret;
EBUG_ON(btree_iter_type(iter) != BTREE_ITER_NODES);
- bch2_btree_iter_checks(iter);
+ bch2_btree_iter_verify(iter);
/* already got to end? */
if (!btree_iter_node(iter, iter->level))
if (btree_node_read_locked(iter, iter->level))
btree_node_unlock(iter, iter->level);
- iter->pos = bkey_successor(iter->pos);
+ iter->pos = iter->real_pos = bkey_successor(iter->pos);
iter->level = iter->min_depth;
btree_iter_set_dirty(iter, BTREE_ITER_NEED_TRAVERSE);
b = iter->l[iter->level].b;
}
- iter->pos = b->key.k.p;
+ iter->pos = iter->real_pos = b->key.k.p;
iter->uptodate = BTREE_ITER_UPTODATE;
bch2_btree_iter_verify(iter);
btree_iter_set_dirty(iter, BTREE_ITER_NEED_PEEK);
}
-void __bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos new_pos,
- bool strictly_greater)
+static void btree_iter_set_search_pos(struct btree_iter *iter, struct bpos new_pos)
{
- struct bpos old = btree_iter_search_key(iter);
- int cmp;
+ int cmp = bkey_cmp(new_pos, iter->real_pos);
- iter->flags &= ~BTREE_ITER_IS_EXTENTS;
- iter->flags |= strictly_greater ? BTREE_ITER_IS_EXTENTS : 0;
+ iter->real_pos = new_pos;
+
+ btree_iter_pos_changed(iter, cmp);
+ bch2_btree_iter_verify(iter);
+}
+
+void __bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos new_pos,
+ bool strictly_greater)
+{
bkey_init(&iter->k);
iter->k.p = iter->pos = new_pos;
- cmp = bkey_cmp(btree_iter_search_key(iter), old);
+ iter->flags &= ~BTREE_ITER_IS_EXTENTS;
+ iter->flags |= strictly_greater ? BTREE_ITER_IS_EXTENTS : 0;
- btree_iter_pos_changed(iter, cmp);
+ btree_iter_set_search_pos(iter, btree_iter_search_key(iter));
}
void bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos new_pos)
{
- int cmp = bkey_cmp(new_pos, iter->pos);
-
- bkey_init(&iter->k);
- iter->k.p = iter->pos = new_pos;
-
- btree_iter_pos_changed(iter, cmp);
+ __bch2_btree_iter_set_pos(iter, new_pos,
+ (iter->flags & BTREE_ITER_IS_EXTENTS) != 0);
}
static inline bool bch2_btree_iter_advance_pos(struct btree_iter *iter)
{
struct bpos pos = iter->k.p;
+ bool ret = bkey_cmp(pos, POS_MAX) != 0;
- if (unlikely(!bkey_cmp(pos, POS_MAX)))
- return false;
-
- if (!(iter->flags & BTREE_ITER_IS_EXTENTS))
+ if (ret && !(iter->flags & BTREE_ITER_IS_EXTENTS))
pos = bkey_successor(pos);
bch2_btree_iter_set_pos(iter, pos);
- return true;
+ return ret;
}
static inline bool bch2_btree_iter_rewind_pos(struct btree_iter *iter)
{
struct bpos pos = bkey_start_pos(&iter->k);
+ bool ret = bkey_cmp(pos, POS_MIN) != 0;
- if (unlikely(!bkey_cmp(pos, POS_MIN)))
- return false;
-
- if (!(iter->flags & BTREE_ITER_IS_EXTENTS))
+ if (ret && !(iter->flags & BTREE_ITER_IS_EXTENTS))
pos = bkey_predecessor(pos);
bch2_btree_iter_set_pos(iter, pos);
- return true;
+ return ret;
}
static inline bool btree_iter_set_pos_to_next_leaf(struct btree_iter *iter)
struct bpos next_pos = iter->l[0].b->key.k.p;
bool ret = bkey_cmp(next_pos, POS_MAX) != 0;
- if (ret && !(iter->flags & BTREE_ITER_IS_EXTENTS))
- next_pos = bkey_successor(next_pos);
+ /*
+ * Typically, we don't want to modify iter->pos here, since that
+ * indicates where we searched from - unless we got to the end of the
+ * btree, in that case we want iter->pos to reflect that:
+ */
+ if (ret)
+ btree_iter_set_search_pos(iter, bkey_successor(next_pos));
+ else
+ bch2_btree_iter_set_pos(iter, POS_MAX);
- bch2_btree_iter_set_pos(iter, next_pos);
return ret;
}
struct bpos next_pos = iter->l[0].b->data->min_key;
bool ret = bkey_cmp(next_pos, POS_MIN) != 0;
- if (ret) {
- next_pos = bkey_predecessor(next_pos);
-
- if (iter->flags & BTREE_ITER_IS_EXTENTS)
- next_pos = bkey_predecessor(next_pos);
- }
+ if (ret)
+ btree_iter_set_search_pos(iter, bkey_predecessor(next_pos));
+ else
+ bch2_btree_iter_set_pos(iter, POS_MIN);
- bch2_btree_iter_set_pos(iter, next_pos);
return ret;
}
int ret;
EBUG_ON(btree_iter_type(iter) != BTREE_ITER_KEYS);
- bch2_btree_iter_checks(iter);
+ bch2_btree_iter_verify(iter);
+ bch2_btree_iter_verify_entry_exit(iter);
+
+ btree_iter_set_search_pos(iter, btree_iter_search_key(iter));
if (iter->uptodate == BTREE_ITER_UPTODATE &&
!bkey_deleted(&iter->k))
if (bkey_cmp(bkey_start_pos(k.k), iter->pos) > 0)
iter->pos = bkey_start_pos(k.k);
+ iter->real_pos = k.k->p;
+
iter->uptodate = BTREE_ITER_UPTODATE;
- bch2_btree_iter_verify_level(iter, 0);
+ bch2_btree_iter_verify_entry_exit(iter);
+ bch2_btree_iter_verify(iter);
return k;
}
int ret;
EBUG_ON(btree_iter_type(iter) != BTREE_ITER_KEYS);
- bch2_btree_iter_checks(iter);
+ bch2_btree_iter_verify(iter);
while (1) {
ret = bch2_btree_iter_traverse(iter);
}
/*
- * iter->pos should always be equal to the key we just
- * returned - except extents can straddle iter->pos:
+ * iter->pos should be mononotically increasing, and always be equal to
+ * the key we just returned - except extents can straddle iter->pos:
*/
if (bkey_cmp(bkey_start_pos(k.k), iter->pos) > 0)
iter->pos = bkey_start_pos(k.k);
int ret;
EBUG_ON(btree_iter_type(iter) != BTREE_ITER_KEYS);
- bch2_btree_iter_checks(iter);
+ bch2_btree_iter_verify(iter);
+ bch2_btree_iter_verify_entry_exit(iter);
+
+ btree_iter_set_search_pos(iter, iter->pos);
if (iter->uptodate == BTREE_ITER_UPTODATE &&
!bkey_deleted(&iter->k))
while (1) {
ret = bch2_btree_iter_traverse(iter);
- if (unlikely(ret))
- return bkey_s_c_err(ret);
+ if (unlikely(ret)) {
+ k = bkey_s_c_err(ret);
+ goto no_key;
+ }
k = __btree_iter_peek(iter, l);
if (!k.k ||
if (likely(k.k))
break;
- if (!btree_iter_set_pos_to_prev_leaf(iter))
- return bkey_s_c_null;
+ if (!btree_iter_set_pos_to_prev_leaf(iter)) {
+ k = bkey_s_c_null;
+ goto no_key;
+ }
}
EBUG_ON(bkey_cmp(bkey_start_pos(k.k), pos) > 0);
/* Extents can straddle iter->pos: */
if (bkey_cmp(k.k->p, pos) < 0)
iter->pos = k.k->p;
- iter->uptodate = BTREE_ITER_UPTODATE;
-
- bch2_btree_iter_verify_level(iter, 0);
+ iter->real_pos = k.k->p;
+ iter->uptodate = BTREE_ITER_UPTODATE;
+out:
+ bch2_btree_iter_verify_entry_exit(iter);
+ bch2_btree_iter_verify(iter);
return k;
+no_key:
+ /*
+ * __btree_iter_peek() may have set iter->k to a key we didn't want, and
+ * then we errored going to the previous leaf - make sure it's
+ * consistent with iter->pos:
+ */
+ BUG_ON(bkey_cmp(pos, iter->pos) &&
+ bkey_cmp(iter->pos, POS_MIN));
+ bkey_init(&iter->k);
+ iter->k.p = iter->pos;
+ goto out;
}
/**
*/
EBUG_ON(bkey_cmp(k.k->p, iter->pos) <= 0);
- bch2_btree_iter_verify_level(iter, 0);
+ bch2_btree_iter_verify(iter);
return k;
}
iter->k = n;
iter->uptodate = BTREE_ITER_UPTODATE;
- bch2_btree_iter_verify_level(iter, 0);
+ bch2_btree_iter_verify_entry_exit(iter);
+ bch2_btree_iter_verify(iter);
+
return (struct bkey_s_c) { &iter->k, NULL };
}
int ret;
EBUG_ON(btree_iter_type(iter) != BTREE_ITER_KEYS);
- bch2_btree_iter_checks(iter);
+ bch2_btree_iter_verify(iter);
+ bch2_btree_iter_verify_entry_exit(iter);
+
+ btree_iter_set_search_pos(iter, btree_iter_search_key(iter));
if (iter->uptodate == BTREE_ITER_UPTODATE)
return btree_iter_peek_uptodate(iter);
}
iter->uptodate = BTREE_ITER_UPTODATE;
- bch2_btree_iter_verify_level(iter, 0);
+ bch2_btree_iter_verify_entry_exit(iter);
+ bch2_btree_iter_verify(iter);
return k;
}
int ret;
EBUG_ON(btree_iter_type(iter) != BTREE_ITER_CACHED);
- bch2_btree_iter_checks(iter);
+ bch2_btree_iter_verify(iter);
ret = bch2_btree_iter_traverse(iter);
if (unlikely(ret))
bkey_init(&iter->k);
iter->k.p = pos;
iter->flags = flags;
+ iter->real_pos = btree_iter_search_key(iter);
iter->uptodate = BTREE_ITER_NEED_TRAVERSE;
iter->btree_id = btree_id;
iter->level = 0;
if (best &&
bkey_cmp(bpos_diff(best->pos, pos),
- bpos_diff(iter->pos, pos)) < 0)
+ bpos_diff(iter->real_pos, pos)) < 0)
continue;
best = iter;