return 0;
}
-int bch2_alloc_replay_key(struct bch_fs *c, struct bkey_i *k)
+enum alloc_write_ret {
+ ALLOC_WROTE,
+ ALLOC_NOWROTE,
+ ALLOC_END,
+};
+
+static int bch2_alloc_write_key(struct btree_trans *trans,
+ struct btree_iter *iter,
+ unsigned flags)
{
- struct btree_trans trans;
- struct btree_iter *iter;
+ struct bch_fs *c = trans->c;
+ struct bkey_s_c k;
struct bch_dev *ca;
+ struct bucket_array *ba;
+ struct bucket *g;
+ struct bucket_mark m;
+ struct bkey_alloc_unpacked old_u, new_u;
+ __BKEY_PADDED(k, 8) alloc_key; /* hack: */
+ struct bkey_i_alloc *a;
int ret;
+retry:
+ k = bch2_btree_iter_peek_slot(iter);
+ ret = bkey_err(k);
+ if (ret)
+ goto err;
- if (k->k.p.inode >= c->sb.nr_devices ||
- !c->devs[k->k.p.inode])
- return 0;
-
- ca = bch_dev_bkey_exists(c, k->k.p.inode);
+ old_u = bch2_alloc_unpack(k);
- if (k->k.p.offset >= ca->mi.nbuckets)
- return 0;
+ if (iter->pos.inode >= c->sb.nr_devices ||
+ !c->devs[iter->pos.inode])
+ return ALLOC_END;
- bch2_trans_init(&trans, c, 0, 0);
+ percpu_down_read(&c->mark_lock);
+ ca = bch_dev_bkey_exists(c, iter->pos.inode);
+ ba = bucket_array(ca);
- iter = bch2_trans_get_iter(&trans, BTREE_ID_ALLOC, k->k.p,
- BTREE_ITER_INTENT);
+ if (iter->pos.offset >= ba->nbuckets) {
+ percpu_up_read(&c->mark_lock);
+ return ALLOC_END;
+ }
- ret = bch2_btree_iter_traverse(iter);
- if (ret)
- goto err;
+ g = &ba->b[iter->pos.offset];
+ m = READ_ONCE(g->mark);
+ new_u = alloc_mem_to_key(g, m);
+ percpu_up_read(&c->mark_lock);
- /* check buckets_written with btree node locked: */
- if (test_bit(k->k.p.offset, ca->buckets_written)) {
- ret = 0;
- goto err;
- }
+ if (!bkey_alloc_unpacked_cmp(old_u, new_u))
+ return ALLOC_NOWROTE;
- bch2_trans_update(&trans, BTREE_INSERT_ENTRY(iter, k));
+ a = bkey_alloc_init(&alloc_key.k);
+ a->k.p = iter->pos;
+ bch2_alloc_pack(a, new_u);
- ret = bch2_trans_commit(&trans, NULL, NULL,
+ bch2_trans_update(trans, BTREE_INSERT_ENTRY(iter, &a->k_i));
+ ret = bch2_trans_commit(trans, NULL, NULL,
+ BTREE_INSERT_ATOMIC|
BTREE_INSERT_NOFAIL|
- BTREE_INSERT_LAZY_RW|
- BTREE_INSERT_JOURNAL_REPLAY|
- BTREE_INSERT_NOMARK);
+ BTREE_INSERT_NOMARK|
+ flags);
err:
- bch2_trans_exit(&trans);
+ if (ret == -EINTR)
+ goto retry;
return ret;
}
{
struct btree_trans trans;
struct btree_iter *iter;
- struct bucket_array *buckets;
struct bch_dev *ca;
- struct bucket *g;
- struct bucket_mark m, new;
- struct bkey_alloc_unpacked old_u, new_u;
- __BKEY_PADDED(k, 8) alloc_key; /* hack: */
- struct bkey_i_alloc *a;
- struct bkey_s_c k;
unsigned i;
- size_t b;
int ret = 0;
BUG_ON(BKEY_ALLOC_VAL_U64s_MAX > 8);
BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
for_each_rw_member(ca, c, i) {
- down_read(&ca->bucket_lock);
-restart:
- buckets = bucket_array(ca);
-
- for (b = buckets->first_bucket;
- b < buckets->nbuckets;
- b++) {
- if (!buckets->b[b].mark.dirty)
- continue;
-
- bch2_btree_iter_set_pos(iter, POS(i, b));
- k = bch2_btree_iter_peek_slot(iter);
- ret = bkey_err(k);
- if (ret)
- goto err;
-
- old_u = bch2_alloc_unpack(k);
-
- percpu_down_read(&c->mark_lock);
- g = bucket(ca, b);
- m = READ_ONCE(g->mark);
- new_u = alloc_mem_to_key(g, m);
- percpu_up_read(&c->mark_lock);
+ unsigned first_bucket;
- if (!m.dirty)
- continue;
-
- if ((flags & BTREE_INSERT_LAZY_RW) &&
- percpu_ref_is_zero(&c->writes)) {
- up_read(&ca->bucket_lock);
- bch2_trans_unlock(&trans);
-
- ret = bch2_fs_read_write_early(c);
- down_read(&ca->bucket_lock);
-
- if (ret)
- goto err;
- goto restart;
- }
+ percpu_down_read(&c->mark_lock);
+ first_bucket = bucket_array(ca)->first_bucket;
+ percpu_up_read(&c->mark_lock);
- a = bkey_alloc_init(&alloc_key.k);
- a->k.p = iter->pos;
- bch2_alloc_pack(a, new_u);
+ bch2_btree_iter_set_pos(iter, POS(i, first_bucket));
- bch2_trans_update(&trans, BTREE_INSERT_ENTRY(iter, &a->k_i));
- ret = bch2_trans_commit(&trans, NULL, NULL,
- BTREE_INSERT_NOFAIL|
- BTREE_INSERT_NOMARK|
- flags);
-err:
- if (ret && !test_bit(BCH_FS_EMERGENCY_RO, &c->flags)) {
- bch_err(c, "error %i writing alloc info", ret);
- printk(KERN_CONT "dev %llu bucket %llu\n",
- iter->pos.inode, iter->pos.offset);
- printk(KERN_CONT "gen %u -> %u\n", old_u.gen, new_u.gen);
-#define x(_name, _bits) printk(KERN_CONT #_name " %u -> %u\n", old_u._name, new_u._name);
- BCH_ALLOC_FIELDS()
-#undef x
- }
- if (ret)
+ while (1) {
+ ret = bch2_alloc_write_key(&trans, iter, flags);
+ if (ret < 0 || ret == ALLOC_END)
break;
-
- new = m;
- new.dirty = false;
- atomic64_cmpxchg(&g->_mark.v, m.v.counter, new.v.counter);
-
- if (ca->buckets_written)
- set_bit(b, ca->buckets_written);
-
- bch2_trans_cond_resched(&trans);
- *wrote = true;
+ if (ret == ALLOC_WROTE)
+ *wrote = true;
+ bch2_btree_iter_next_slot(iter);
}
- up_read(&ca->bucket_lock);
- if (ret) {
+ if (ret < 0) {
percpu_ref_put(&ca->io_ref);
break;
}
bch2_trans_exit(&trans);
- return ret;
+ return ret < 0 ? ret : 0;
+}
+
+int bch2_alloc_replay_key(struct bch_fs *c, struct bkey_i *k)
+{
+ struct btree_trans trans;
+ struct btree_iter *iter;
+ int ret;
+
+ bch2_trans_init(&trans, c, 0, 0);
+
+ iter = bch2_trans_get_iter(&trans, BTREE_ID_ALLOC, k->k.p,
+ BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
+
+ ret = bch2_alloc_write_key(&trans, iter,
+ BTREE_INSERT_NOFAIL|
+ BTREE_INSERT_LAZY_RW|
+ BTREE_INSERT_JOURNAL_REPLAY|
+ BTREE_INSERT_NOMARK);
+ bch2_trans_exit(&trans);
+ return ret < 0 ? ret : 0;
}
/* Bucket IO clocks: */
if (!top->nr)
heap_pop(&ca->alloc_heap, e, bucket_alloc_cmp, NULL);
- /* with btree still locked: */
- if (ca->buckets_written)
- set_bit(b, ca->buckets_written);
-
/*
* Make sure we flush the last journal entry that updated this
* bucket (i.e. deleting the last reference) before writing to
"type %u gen %u",
k.k->type, ptr->gen)) {
g2->_mark.gen = g->_mark.gen = ptr->gen;
- g2->_mark.dirty = g->_mark.dirty = true;
g2->gen_valid = g->gen_valid = true;
}
"%u ptr gen in the future: %u > %u",
k.k->type, ptr->gen, g->mark.gen)) {
g2->_mark.gen = g->_mark.gen = ptr->gen;
- g2->_mark.dirty = g->_mark.dirty = true;
g2->gen_valid = g->gen_valid = true;
set_bit(BCH_FS_FIXED_GENS, &c->flags);
}
": got %u, should be %u", i, b, \
dst->b[b].mark._f, src->b[b].mark._f); \
dst->b[b]._mark._f = src->b[b].mark._f; \
- dst->b[b]._mark.dirty = true; \
}
#define copy_dev_field(_f, _msg, ...) \
copy_field(_f, "dev %u has wrong " _msg, i, ##__VA_ARGS__)
copy_bucket_field(dirty_sectors);
copy_bucket_field(cached_sectors);
- if (dst->b[b].oldest_gen != src->b[b].oldest_gen) {
- dst->b[b].oldest_gen = src->b[b].oldest_gen;
- dst->b[b]._mark.dirty = true;
- }
+ dst->b[b].oldest_gen = src->b[b].oldest_gen;
}
};
BUG_ON(!is_available_bucket(new));
new.owned_by_allocator = true;
- new.dirty = true;
new.data_type = 0;
new.cached_sectors = 0;
new.dirty_sectors = 0;
type != BCH_DATA_JOURNAL);
old = bucket_cmpxchg(g, new, ({
- new.dirty = true;
new.data_type = type;
overflow = checked_add(new.dirty_sectors, sectors);
}));
struct bucket_mark new, old;
old = bucket_data_cmpxchg(c, ca, fs_usage, g, new, ({
- new.dirty = true;
new.stripe = enabled;
if (journal_seq) {
new.journal_seq_valid = 1;
do {
new.v.counter = old.v.counter = v;
- new.dirty = true;
-
/*
* Check this after reading bucket mark to guard against
* the allocator invalidating a bucket after we've already
{
struct bucket_array *buckets = NULL, *old_buckets = NULL;
unsigned long *buckets_nouse = NULL;
- unsigned long *buckets_written = NULL;
alloc_fifo free[RESERVE_NR];
alloc_fifo free_inc;
alloc_heap alloc_heap;
!(buckets_nouse = kvpmalloc(BITS_TO_LONGS(nbuckets) *
sizeof(unsigned long),
GFP_KERNEL|__GFP_ZERO)) ||
- !(buckets_written = kvpmalloc(BITS_TO_LONGS(nbuckets) *
- sizeof(unsigned long),
- GFP_KERNEL|__GFP_ZERO)) ||
!init_fifo(&free[RESERVE_BTREE], btree_reserve, GFP_KERNEL) ||
!init_fifo(&free[RESERVE_MOVINGGC],
copygc_reserve, GFP_KERNEL) ||
memcpy(buckets_nouse,
ca->buckets_nouse,
BITS_TO_LONGS(n) * sizeof(unsigned long));
- memcpy(buckets_written,
- ca->buckets_written,
- BITS_TO_LONGS(n) * sizeof(unsigned long));
}
rcu_assign_pointer(ca->buckets[0], buckets);
buckets = old_buckets;
swap(ca->buckets_nouse, buckets_nouse);
- swap(ca->buckets_written, buckets_written);
if (resize)
percpu_up_write(&c->mark_lock);
free_fifo(&free[i]);
kvpfree(buckets_nouse,
BITS_TO_LONGS(nbuckets) * sizeof(unsigned long));
- kvpfree(buckets_written,
- BITS_TO_LONGS(nbuckets) * sizeof(unsigned long));
if (buckets)
call_rcu(&old_buckets->rcu, buckets_free_rcu);
free_fifo(&ca->free_inc);
for (i = 0; i < RESERVE_NR; i++)
free_fifo(&ca->free[i]);
- kvpfree(ca->buckets_written,
- BITS_TO_LONGS(ca->mi.nbuckets) * sizeof(unsigned long));
kvpfree(ca->buckets_nouse,
BITS_TO_LONGS(ca->mi.nbuckets) * sizeof(unsigned long));
kvpfree(rcu_dereference_protected(ca->buckets[0], 1),
projects / linux.git / commitdiff