struct percpu_rw_semaphore mark_lock;
+ seqcount_t usage_lock;
+ struct bch_fs_usage *usage_base;
struct bch_fs_usage __percpu *usage[2];
+ struct bch_fs_usage __percpu *usage_gc;
+ u64 __percpu *online_reserved;
/* single element mempool: */
struct mutex usage_scratch_lock;
- struct bch_fs_usage *usage_scratch;
+ struct bch_fs_usage_online *usage_scratch;
/*
* When we invalidate buckets, we use both the priority and the amount
ca->usage[1] = NULL;
}
- free_percpu(c->usage[1]);
- c->usage[1] = NULL;
+ free_percpu(c->usage_gc);
+ c->usage_gc = NULL;
}
static int bch2_gc_done(struct bch_fs *c,
}
};
+ for (i = 0; i < ARRAY_SIZE(c->usage); i++)
+ bch2_fs_usage_acc_to_base(c, i);
+
bch2_dev_usage_from_buckets(c);
{
unsigned nr = fs_usage_u64s(c);
- struct bch_fs_usage *dst = (void *)
- bch2_acc_percpu_u64s((void *) c->usage[0], nr);
+ struct bch_fs_usage *dst = c->usage_base;
struct bch_fs_usage *src = (void *)
- bch2_acc_percpu_u64s((void *) c->usage[1], nr);
+ bch2_acc_percpu_u64s((void *) c->usage_gc, nr);
copy_fs_field(hidden, "hidden");
copy_fs_field(btree, "btree");
*/
gc_pos_set(c, gc_phase(GC_PHASE_START));
- BUG_ON(c->usage[1]);
+ BUG_ON(c->usage_gc);
- c->usage[1] = __alloc_percpu_gfp(fs_usage_u64s(c) * sizeof(u64),
+ c->usage_gc = __alloc_percpu_gfp(fs_usage_u64s(c) * sizeof(u64),
sizeof(u64), GFP_KERNEL);
- if (!c->usage[1])
+ if (!c->usage_gc)
return -ENOMEM;
for_each_member_device(ca, c, i) {
ret = -EINVAL;
}
- percpu_down_write(&c->mark_lock);
+ if (!ret) {
+ bch2_journal_block(&c->journal);
- if (!ret)
+ percpu_down_write(&c->mark_lock);
ret = bch2_gc_done(c, initial, metadata_only);
+ bch2_journal_unblock(&c->journal);
+ } else {
+ percpu_down_write(&c->mark_lock);
+ }
+
/* Indicates that gc is no longer in progress: */
__gc_pos_set(c, gc_phase(GC_PHASE_NOT_RUNNING));
{
struct bch_fs *c = as->c;
struct btree *old = btree_node_root(c, b);
- struct bch_fs_usage *fs_usage;
+ struct bch_fs_usage_online *fs_usage;
__bch2_btree_set_root_inmem(c, b);
fs_usage = bch2_fs_usage_scratch_get(c);
bch2_mark_key_locked(c, bkey_i_to_s_c(&b->key),
- true, 0, fs_usage, 0, 0);
+ true, 0, &fs_usage->u, 0, 0);
if (gc_visited(c, gc_pos_btree_root(b->btree_id)))
bch2_mark_key_locked(c, bkey_i_to_s_c(&b->key),
true, 0, NULL, 0,
if (old && !btree_node_fake(old))
bch2_btree_node_free_index(as, NULL,
bkey_i_to_s_c(&old->key),
- fs_usage);
- bch2_fs_usage_apply(c, fs_usage, &as->reserve->disk_res);
+ &fs_usage->u);
+ bch2_fs_usage_apply(c, fs_usage, &as->reserve->disk_res, 0);
bch2_fs_usage_scratch_put(c, fs_usage);
percpu_up_read(&c->mark_lock);
struct btree_node_iter *node_iter)
{
struct bch_fs *c = as->c;
- struct bch_fs_usage *fs_usage;
+ struct bch_fs_usage_online *fs_usage;
struct bkey_packed *k;
struct bkey tmp;
fs_usage = bch2_fs_usage_scratch_get(c);
bch2_mark_key_locked(c, bkey_i_to_s_c(insert),
- true, 0, fs_usage, 0, 0);
+ true, 0, &fs_usage->u, 0, 0);
if (gc_visited(c, gc_pos_btree_node(b)))
bch2_mark_key_locked(c, bkey_i_to_s_c(insert),
if (k && !bkey_cmp_packed(b, k, &insert->k))
bch2_btree_node_free_index(as, b,
bkey_disassemble(b, k, &tmp),
- fs_usage);
+ &fs_usage->u);
- bch2_fs_usage_apply(c, fs_usage, &as->reserve->disk_res);
+ bch2_fs_usage_apply(c, fs_usage, &as->reserve->disk_res, 0);
bch2_fs_usage_scratch_put(c, fs_usage);
percpu_up_read(&c->mark_lock);
bkey_copy(&b->key, &new_key->k_i);
}
} else {
- struct bch_fs_usage *fs_usage;
+ struct bch_fs_usage_online *fs_usage;
BUG_ON(btree_node_root(c, b) != b);
fs_usage = bch2_fs_usage_scratch_get(c);
bch2_mark_key_locked(c, bkey_i_to_s_c(&new_key->k_i),
- true, 0, fs_usage, 0, 0);
+ true, 0, &fs_usage->u, 0, 0);
if (gc_visited(c, gc_pos_btree_root(b->btree_id)))
bch2_mark_key_locked(c, bkey_i_to_s_c(&new_key->k_i),
true, 0, NULL, 0,
bch2_btree_node_free_index(as, NULL,
bkey_i_to_s_c(&b->key),
- fs_usage);
- bch2_fs_usage_apply(c, fs_usage, &as->reserve->disk_res);
+ &fs_usage->u);
+ bch2_fs_usage_apply(c, fs_usage, &as->reserve->disk_res, 0);
bch2_fs_usage_scratch_put(c, fs_usage);
percpu_up_read(&c->mark_lock);
struct btree_insert_entry **stopped_at)
{
struct bch_fs *c = trans->c;
- struct bch_fs_usage *fs_usage = NULL;
+ struct bch_fs_usage_online *fs_usage = NULL;
struct btree_insert_entry *i;
struct btree_iter *linked;
int ret;
if (likely(!(trans->flags & BTREE_INSERT_NOMARK))) {
trans_for_each_update_iter(trans, i)
- bch2_mark_update(trans, i, fs_usage, 0);
+ bch2_mark_update(trans, i, &fs_usage->u, 0);
if (fs_usage)
bch2_trans_fs_usage_apply(trans, fs_usage);
unsigned i;
percpu_down_write(&c->mark_lock);
- usage = (void *) bch2_acc_percpu_u64s((void *) c->usage[0],
- fs_usage_u64s(c));
+ usage = c->usage_base;
+
+ for (i = 0; i < ARRAY_SIZE(c->usage); i++)
+ bch2_fs_usage_acc_to_base(c, i);
for (i = 0; i < BCH_REPLICAS_MAX; i++)
usage->reserved += usage->persistent_reserved[i];
percpu_up_write(&c->mark_lock);
}
-void bch2_fs_usage_scratch_put(struct bch_fs *c, struct bch_fs_usage *fs_usage)
+void bch2_fs_usage_scratch_put(struct bch_fs *c, struct bch_fs_usage_online *fs_usage)
{
if (fs_usage == c->usage_scratch)
mutex_unlock(&c->usage_scratch_lock);
kfree(fs_usage);
}
-struct bch_fs_usage *bch2_fs_usage_scratch_get(struct bch_fs *c)
+struct bch_fs_usage_online *bch2_fs_usage_scratch_get(struct bch_fs *c)
{
- struct bch_fs_usage *ret;
- unsigned bytes = fs_usage_u64s(c) * sizeof(u64);
-
- ret = kzalloc(bytes, GFP_NOWAIT);
+ struct bch_fs_usage_online *ret;
+ unsigned bytes = sizeof(struct bch_fs_usage_online) + sizeof(u64) *
+ READ_ONCE(c->replicas.nr);
+ ret = kzalloc(bytes, GFP_NOWAIT|__GFP_NOWARN);
if (ret)
return ret;
return ret;
}
-struct bch_fs_usage *bch2_fs_usage_read(struct bch_fs *c)
+static inline struct bch_fs_usage *fs_usage_ptr(struct bch_fs *c,
+ unsigned journal_seq,
+ bool gc)
{
- struct bch_fs_usage *ret;
- unsigned v, u64s = fs_usage_u64s(c);
-retry:
- ret = kzalloc(u64s * sizeof(u64), GFP_NOFS);
- if (unlikely(!ret))
- return NULL;
+ return this_cpu_ptr(gc
+ ? c->usage_gc
+ : c->usage[journal_seq & 1]);
+}
+
+u64 bch2_fs_usage_read_one(struct bch_fs *c, u64 *v)
+{
+ ssize_t offset = v - (u64 *) c->usage_base;
+ unsigned seq;
+ u64 ret;
+
+ BUG_ON(offset < 0 || offset >= fs_usage_u64s(c));
+ percpu_rwsem_assert_held(&c->mark_lock);
+
+ do {
+ seq = read_seqcount_begin(&c->usage_lock);
+ ret = *v +
+ percpu_u64_get((u64 __percpu *) c->usage[0] + offset) +
+ percpu_u64_get((u64 __percpu *) c->usage[1] + offset);
+ } while (read_seqcount_retry(&c->usage_lock, seq));
+
+ return ret;
+}
+
+struct bch_fs_usage_online *bch2_fs_usage_read(struct bch_fs *c)
+{
+ struct bch_fs_usage_online *ret;
+ unsigned seq, i, u64s;
percpu_down_read(&c->mark_lock);
- v = fs_usage_u64s(c);
- if (unlikely(u64s != v)) {
- u64s = v;
+ ret = kmalloc(sizeof(struct bch_fs_usage_online) +
+ sizeof(u64) + c->replicas.nr, GFP_NOFS);
+ if (unlikely(!ret)) {
percpu_up_read(&c->mark_lock);
- kfree(ret);
- goto retry;
+ return NULL;
}
- acc_u64s_percpu((u64 *) ret, (u64 __percpu *) c->usage[0], u64s);
+ ret->online_reserved = percpu_u64_get(c->online_reserved);
+
+ u64s = fs_usage_u64s(c);
+ do {
+ seq = read_seqcount_begin(&c->usage_lock);
+ memcpy(&ret->u, c->usage_base, u64s * sizeof(u64));
+ for (i = 0; i < ARRAY_SIZE(c->usage); i++)
+ acc_u64s_percpu((u64 *) &ret->u, (u64 __percpu *) c->usage[i], u64s);
+ } while (read_seqcount_retry(&c->usage_lock, seq));
return ret;
}
+void bch2_fs_usage_acc_to_base(struct bch_fs *c, unsigned idx)
+{
+ unsigned u64s = fs_usage_u64s(c);
+
+ BUG_ON(idx >= ARRAY_SIZE(c->usage));
+
+ preempt_disable();
+ write_seqcount_begin(&c->usage_lock);
+
+ acc_u64s_percpu((u64 *) c->usage_base,
+ (u64 __percpu *) c->usage[idx], u64s);
+ percpu_memset(c->usage[idx], 0, u64s * sizeof(u64));
+
+ write_seqcount_end(&c->usage_lock);
+ preempt_enable();
+}
+
+void bch2_fs_usage_to_text(struct printbuf *out,
+ struct bch_fs *c,
+ struct bch_fs_usage_online *fs_usage)
+{
+ unsigned i;
+
+ pr_buf(out, "capacity:\t\t\t%llu\n", c->capacity);
+
+ pr_buf(out, "hidden:\t\t\t\t%llu\n",
+ fs_usage->u.hidden);
+ pr_buf(out, "data:\t\t\t\t%llu\n",
+ fs_usage->u.data);
+ pr_buf(out, "cached:\t\t\t\t%llu\n",
+ fs_usage->u.cached);
+ pr_buf(out, "reserved:\t\t\t%llu\n",
+ fs_usage->u.reserved);
+ pr_buf(out, "nr_inodes:\t\t\t%llu\n",
+ fs_usage->u.nr_inodes);
+ pr_buf(out, "online reserved:\t\t%llu\n",
+ fs_usage->online_reserved);
+
+ for (i = 0;
+ i < ARRAY_SIZE(fs_usage->u.persistent_reserved);
+ i++) {
+ pr_buf(out, "%u replicas:\n", i + 1);
+ pr_buf(out, "\treserved:\t\t%llu\n",
+ fs_usage->u.persistent_reserved[i]);
+ }
+
+ for (i = 0; i < c->replicas.nr; i++) {
+ struct bch_replicas_entry *e =
+ cpu_replicas_entry(&c->replicas, i);
+
+ pr_buf(out, "\t");
+ bch2_replicas_entry_to_text(out, e);
+ pr_buf(out, ":\t%llu\n", fs_usage->u.replicas[i]);
+ }
+}
+
#define RESERVE_FACTOR 6
static u64 reserve_factor(u64 r)
return (r << RESERVE_FACTOR) / ((1 << RESERVE_FACTOR) + 1);
}
-u64 bch2_fs_sectors_used(struct bch_fs *c, struct bch_fs_usage *fs_usage)
+u64 bch2_fs_sectors_used(struct bch_fs *c, struct bch_fs_usage_online *fs_usage)
{
- return min(fs_usage->hidden +
- fs_usage->btree +
- fs_usage->data +
- reserve_factor(fs_usage->reserved +
+ return min(fs_usage->u.hidden +
+ fs_usage->u.btree +
+ fs_usage->u.data +
+ reserve_factor(fs_usage->u.reserved +
fs_usage->online_reserved),
c->capacity);
}
u64 data, reserved;
ret.capacity = c->capacity -
- percpu_u64_get(&c->usage[0]->hidden);
+ bch2_fs_usage_read_one(c, &c->usage_base->hidden);
- data = percpu_u64_get(&c->usage[0]->data) +
- percpu_u64_get(&c->usage[0]->btree);
- reserved = percpu_u64_get(&c->usage[0]->reserved) +
- percpu_u64_get(&c->usage[0]->online_reserved);
+ data = bch2_fs_usage_read_one(c, &c->usage_base->data) +
+ bch2_fs_usage_read_one(c, &c->usage_base->btree);
+ reserved = bch2_fs_usage_read_one(c, &c->usage_base->reserved) +
+ percpu_u64_get(c->online_reserved);
ret.used = min(ret.capacity, data + reserve_factor(reserved));
ret.free = ret.capacity - ret.used;
- ret.nr_inodes = percpu_u64_get(&c->usage[0]->nr_inodes);
+ ret.nr_inodes = bch2_fs_usage_read_one(c, &c->usage_base->nr_inodes);
return ret;
}
}
int bch2_fs_usage_apply(struct bch_fs *c,
- struct bch_fs_usage *fs_usage,
- struct disk_reservation *disk_res)
+ struct bch_fs_usage_online *src,
+ struct disk_reservation *disk_res,
+ unsigned journal_seq)
{
- s64 added = fs_usage->data + fs_usage->reserved;
+ struct bch_fs_usage *dst = fs_usage_ptr(c, journal_seq, false);
+ s64 added = src->u.data + src->u.reserved;
s64 should_not_have_added;
int ret = 0;
*/
should_not_have_added = added - (s64) (disk_res ? disk_res->sectors : 0);
if (WARN_ONCE(should_not_have_added > 0,
- "disk usage increased without a reservation")) {
+ "disk usage increased by %lli more than reservation of %llu",
+ added, disk_res ? disk_res->sectors : 0)) {
atomic64_sub(should_not_have_added, &c->sectors_available);
added -= should_not_have_added;
ret = -1;
}
if (added > 0) {
- disk_res->sectors -= added;
- fs_usage->online_reserved -= added;
+ disk_res->sectors -= added;
+ src->online_reserved -= added;
}
+ this_cpu_add(*c->online_reserved, src->online_reserved);
+
preempt_disable();
- acc_u64s((u64 *) this_cpu_ptr(c->usage[0]),
- (u64 *) fs_usage, fs_usage_u64s(c));
+ acc_u64s((u64 *) dst, (u64 *) &src->u, fs_usage_u64s(c));
preempt_enable();
return ret;
account_bucket(fs_usage, dev_usage, bucket_type(new),
1, ca->mi.bucket_size);
- dev_usage->buckets_alloc +=
- (int) new.owned_by_allocator - (int) old.owned_by_allocator;
- dev_usage->buckets_ec +=
- (int) new.stripe - (int) old.stripe;
+ dev_usage->buckets_ec += (int) new.stripe - (int) old.stripe;
dev_usage->buckets_unavailable +=
is_unavailable_bucket(new) - is_unavailable_bucket(old);
{
struct bch_dev *ca;
struct bucket_mark old = { .v.counter = 0 };
- struct bch_fs_usage *fs_usage;
struct bucket_array *buckets;
struct bucket *g;
unsigned i;
int cpu;
- percpu_u64_set(&c->usage[0]->hidden, 0);
-
- /*
- * This is only called during startup, before there's any multithreaded
- * access to c->usage:
- */
- preempt_disable();
- fs_usage = this_cpu_ptr(c->usage[0]);
- preempt_enable();
+ c->usage_base->hidden = 0;
for_each_member_device(ca, c, i) {
for_each_possible_cpu(cpu)
buckets = bucket_array(ca);
for_each_bucket(g, buckets)
- bch2_dev_usage_update(c, ca, fs_usage,
+ bch2_dev_usage_update(c, ca, c->usage_base,
old, g->mark, false);
}
}
size_t b, struct bucket_mark *ret,
bool gc)
{
- struct bch_fs_usage *fs_usage = this_cpu_ptr(c->usage[gc]);
+ struct bch_fs_usage *fs_usage = fs_usage_ptr(c, 0, gc);
struct bucket *g = __bucket(ca, b, gc);
struct bucket_mark old, new;
size_t b, bool owned_by_allocator,
bool gc)
{
- struct bch_fs_usage *fs_usage = this_cpu_ptr(c->usage[gc]);
+ struct bch_fs_usage *fs_usage = fs_usage_ptr(c, 0, gc);
struct bucket *g = __bucket(ca, b, gc);
struct bucket_mark old, new;
old.dirty_sectors, sectors);
if (c)
- bch2_dev_usage_update(c, ca, this_cpu_ptr(c->usage[gc]),
+ bch2_dev_usage_update(c, ca, fs_usage_ptr(c, 0, gc),
old, new, gc);
return 0;
preempt_disable();
if (!fs_usage || gc)
- fs_usage = this_cpu_ptr(c->usage[gc]);
+ fs_usage = fs_usage_ptr(c, journal_seq, gc);
switch (k.k->type) {
case KEY_TYPE_alloc:
}
void bch2_trans_fs_usage_apply(struct btree_trans *trans,
- struct bch_fs_usage *fs_usage)
+ struct bch_fs_usage_online *fs_usage)
{
struct bch_fs *c = trans->c;
struct btree_insert_entry *i;
u64 disk_res_sectors = trans->disk_res ? trans->disk_res->sectors : 0;
char buf[200];
- if (!bch2_fs_usage_apply(c, fs_usage, trans->disk_res) ||
+ if (!bch2_fs_usage_apply(c, fs_usage, trans->disk_res,
+ trans->journal_res.seq) ||
warned_disk_usage ||
xchg(&warned_disk_usage, 1))
return;
/* Disk reservations: */
-void __bch2_disk_reservation_put(struct bch_fs *c, struct disk_reservation *res)
-{
- percpu_down_read(&c->mark_lock);
- this_cpu_sub(c->usage[0]->online_reserved, res->sectors);
- percpu_up_read(&c->mark_lock);
-
- res->sectors = 0;
-}
-
#define SECTORS_CACHE 1024
int bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res,
out:
pcpu->sectors_available -= sectors;
- this_cpu_add(c->usage[0]->online_reserved, sectors);
+ this_cpu_add(*c->online_reserved, sectors);
res->sectors += sectors;
preempt_enable();
(flags & BCH_DISK_RESERVATION_NOFAIL)) {
atomic64_set(&c->sectors_available,
max_t(s64, 0, sectors_available - sectors));
- this_cpu_add(c->usage[0]->online_reserved, sectors);
+ this_cpu_add(*c->online_reserved, sectors);
res->sectors += sectors;
ret = 0;
} else {
READ_ONCE(c->replicas.nr);
}
-void bch2_fs_usage_scratch_put(struct bch_fs *, struct bch_fs_usage *);
-struct bch_fs_usage *bch2_fs_usage_scratch_get(struct bch_fs *);
+void bch2_fs_usage_scratch_put(struct bch_fs *, struct bch_fs_usage_online *);
+struct bch_fs_usage_online *bch2_fs_usage_scratch_get(struct bch_fs *);
-struct bch_fs_usage *bch2_fs_usage_read(struct bch_fs *);
+u64 bch2_fs_usage_read_one(struct bch_fs *, u64 *);
-u64 bch2_fs_sectors_used(struct bch_fs *, struct bch_fs_usage *);
+struct bch_fs_usage_online *bch2_fs_usage_read(struct bch_fs *);
+
+void bch2_fs_usage_acc_to_base(struct bch_fs *, unsigned);
+
+void bch2_fs_usage_to_text(struct printbuf *,
+ struct bch_fs *, struct bch_fs_usage_online *);
+
+u64 bch2_fs_sectors_used(struct bch_fs *, struct bch_fs_usage_online *);
struct bch_fs_usage_short
bch2_fs_usage_read_short(struct bch_fs *);
int bch2_mark_key(struct bch_fs *, struct bkey_s_c,
bool, s64, struct bch_fs_usage *,
u64, unsigned);
-int bch2_fs_usage_apply(struct bch_fs *, struct bch_fs_usage *,
- struct disk_reservation *);
+int bch2_fs_usage_apply(struct bch_fs *, struct bch_fs_usage_online *,
+ struct disk_reservation *, unsigned);
int bch2_mark_overwrite(struct btree_trans *, struct btree_iter *,
struct bkey_s_c, struct bkey_i *,
struct bch_fs_usage *, unsigned);
int bch2_mark_update(struct btree_trans *, struct btree_insert_entry *,
struct bch_fs_usage *, unsigned);
-void bch2_trans_fs_usage_apply(struct btree_trans *, struct bch_fs_usage *);
+void bch2_trans_fs_usage_apply(struct btree_trans *, struct bch_fs_usage_online *);
/* disk reservations: */
-void __bch2_disk_reservation_put(struct bch_fs *, struct disk_reservation *);
-
static inline void bch2_disk_reservation_put(struct bch_fs *c,
struct disk_reservation *res)
{
- if (res->sectors)
- __bch2_disk_reservation_put(c, res);
+ this_cpu_sub(*c->online_reserved, res->sectors);
+ res->sectors = 0;
}
#define BCH_DISK_RESERVATION_NOFAIL (1 << 0)
struct bch_dev_usage {
u64 buckets[BCH_DATA_NR];
- u64 buckets_alloc;
u64 buckets_ec;
u64 buckets_unavailable;
struct bch_fs_usage {
/* all fields are in units of 512 byte sectors: */
-
- u64 online_reserved;
-
- /* fields after online_reserved are cleared/recalculated by gc: */
- u64 gc_start[0];
-
u64 hidden;
u64 btree;
u64 data;
u64 replicas[];
};
+struct bch_fs_usage_online {
+ u64 online_reserved;
+ struct bch_fs_usage u;
+};
+
struct bch_fs_usage_short {
u64 capacity;
u64 used;
}
{
- struct bch_fs_usage *src;
+ struct bch_fs_usage_online *src;
struct bch_ioctl_fs_usage dst = {
.capacity = c->capacity,
};
for (i = 0; i < BCH_REPLICAS_MAX; i++) {
dst.persistent_reserved[i] =
- src->persistent_reserved[i];
+ src->u.persistent_reserved[i];
#if 0
for (j = 0; j < BCH_DATA_NR; j++)
dst.sectors[j][i] = src.replicas[i].data[j];
switch (entry->btree_id) {
case FS_USAGE_RESERVED:
if (entry->level < BCH_REPLICAS_MAX)
- percpu_u64_set(&c->usage[0]->
- persistent_reserved[entry->level],
- le64_to_cpu(u->v));
+ c->usage_base->persistent_reserved[entry->level] =
+ le64_to_cpu(u->v);
break;
case FS_USAGE_INODES:
- percpu_u64_set(&c->usage[0]->nr_inodes,
- le64_to_cpu(u->v));
+ c->usage_base->nr_inodes = le64_to_cpu(u->v);
break;
case FS_USAGE_KEY_VERSION:
atomic64_set(&c->key_version,
// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
+#include "buckets.h"
#include "journal.h"
#include "replicas.h"
#include "super-io.h"
return marked;
}
-static void __replicas_table_update(struct bch_fs_usage __percpu *dst_p,
+static void __replicas_table_update(struct bch_fs_usage *dst,
struct bch_replicas_cpu *dst_r,
- struct bch_fs_usage __percpu *src_p,
+ struct bch_fs_usage *src,
struct bch_replicas_cpu *src_r)
{
- unsigned src_nr = sizeof(struct bch_fs_usage) / sizeof(u64) + src_r->nr;
- struct bch_fs_usage *dst, *src = (void *)
- bch2_acc_percpu_u64s((void *) src_p, src_nr);
int src_idx, dst_idx;
- preempt_disable();
- dst = this_cpu_ptr(dst_p);
- preempt_enable();
-
*dst = *src;
for (src_idx = 0; src_idx < src_r->nr; src_idx++) {
}
}
+static void __replicas_table_update_pcpu(struct bch_fs_usage __percpu *dst_p,
+ struct bch_replicas_cpu *dst_r,
+ struct bch_fs_usage __percpu *src_p,
+ struct bch_replicas_cpu *src_r)
+{
+ unsigned src_nr = sizeof(struct bch_fs_usage) / sizeof(u64) + src_r->nr;
+ struct bch_fs_usage *dst, *src = (void *)
+ bch2_acc_percpu_u64s((void *) src_p, src_nr);
+
+ preempt_disable();
+ dst = this_cpu_ptr(dst_p);
+ preempt_enable();
+
+ __replicas_table_update(dst, dst_r, src, src_r);
+}
+
/*
* Resize filesystem accounting:
*/
static int replicas_table_update(struct bch_fs *c,
struct bch_replicas_cpu *new_r)
{
- struct bch_fs_usage __percpu *new_usage[2] = { NULL, NULL };
- struct bch_fs_usage *new_scratch = NULL;
- unsigned bytes = sizeof(struct bch_fs_usage) +
+ struct bch_fs_usage __percpu *new_usage[2];
+ struct bch_fs_usage_online *new_scratch = NULL;
+ struct bch_fs_usage __percpu *new_gc = NULL;
+ struct bch_fs_usage *new_base = NULL;
+ unsigned i, bytes = sizeof(struct bch_fs_usage) +
+ sizeof(u64) * new_r->nr;
+ unsigned scratch_bytes = sizeof(struct bch_fs_usage_online) +
sizeof(u64) * new_r->nr;
int ret = -ENOMEM;
- if (!(new_usage[0] = __alloc_percpu_gfp(bytes, sizeof(u64),
- GFP_NOIO)) ||
- (c->usage[1] &&
- !(new_usage[1] = __alloc_percpu_gfp(bytes, sizeof(u64),
- GFP_NOIO))) ||
- !(new_scratch = kmalloc(bytes, GFP_NOIO)))
- goto err;
+ memset(new_usage, 0, sizeof(new_usage));
- if (c->usage[0])
- __replicas_table_update(new_usage[0], new_r,
- c->usage[0], &c->replicas);
- if (c->usage[1])
- __replicas_table_update(new_usage[1], new_r,
- c->usage[1], &c->replicas);
+ for (i = 0; i < ARRAY_SIZE(new_usage); i++)
+ if (!(new_usage[i] = __alloc_percpu_gfp(bytes,
+ sizeof(u64), GFP_NOIO)))
+ goto err;
- swap(c->usage[0], new_usage[0]);
- swap(c->usage[1], new_usage[1]);
+ if (!(new_base = kzalloc(bytes, GFP_NOIO)) ||
+ !(new_scratch = kmalloc(scratch_bytes, GFP_NOIO)) ||
+ (c->usage_gc &&
+ !(new_gc = __alloc_percpu_gfp(bytes, sizeof(u64), GFP_NOIO))))
+ goto err;
+
+ for (i = 0; i < ARRAY_SIZE(new_usage); i++)
+ if (c->usage[i])
+ __replicas_table_update_pcpu(new_usage[i], new_r,
+ c->usage[i], &c->replicas);
+ if (c->usage_base)
+ __replicas_table_update(new_base, new_r,
+ c->usage_base, &c->replicas);
+ if (c->usage_gc)
+ __replicas_table_update_pcpu(new_gc, new_r,
+ c->usage_gc, &c->replicas);
+
+ for (i = 0; i < ARRAY_SIZE(new_usage); i++)
+ swap(c->usage[i], new_usage[i]);
+ swap(c->usage_base, new_base);
swap(c->usage_scratch, new_scratch);
+ swap(c->usage_gc, new_gc);
swap(c->replicas, *new_r);
ret = 0;
err:
+ free_percpu(new_gc);
kfree(new_scratch);
free_percpu(new_usage[1]);
free_percpu(new_usage[0]);
+ kfree(new_base);
return ret;
}
lockdep_assert_held(&c->replicas_gc_lock);
mutex_lock(&c->sb_lock);
-
- if (ret)
- goto err;
+ percpu_down_write(&c->mark_lock);
/*
* this is kind of crappy; the replicas gc mechanism needs to be ripped
struct bch_replicas_entry *e =
cpu_replicas_entry(&c->replicas, i);
struct bch_replicas_cpu n;
- u64 v;
- if (__replicas_has_entry(&c->replicas_gc, e))
- continue;
-
- v = percpu_u64_get(&c->usage[0]->replicas[i]);
- if (!v)
- continue;
+ if (!__replicas_has_entry(&c->replicas_gc, e) &&
+ (c->usage_base->replicas[i] ||
+ percpu_u64_get(&c->usage[0]->replicas[i]) ||
+ percpu_u64_get(&c->usage[1]->replicas[i]))) {
+ n = cpu_replicas_add_entry(&c->replicas_gc, e);
+ if (!n.entries) {
+ ret = -ENOSPC;
+ goto err;
+ }
- n = cpu_replicas_add_entry(&c->replicas_gc, e);
- if (!n.entries) {
- ret = -ENOSPC;
- goto err;
+ swap(n, c->replicas_gc);
+ kfree(n.entries);
}
-
- percpu_down_write(&c->mark_lock);
- swap(n, c->replicas_gc);
- percpu_up_write(&c->mark_lock);
-
- kfree(n.entries);
}
if (bch2_cpu_replicas_to_sb_replicas(c, &c->replicas_gc)) {
goto err;
}
- bch2_write_super(c);
-
- /* don't update in memory replicas until changes are persistent */
+ ret = replicas_table_update(c, &c->replicas_gc);
err:
- percpu_down_write(&c->mark_lock);
- if (!ret)
- ret = replicas_table_update(c, &c->replicas_gc);
-
kfree(c->replicas_gc.entries);
c->replicas_gc.entries = NULL;
+
percpu_up_write(&c->mark_lock);
+ if (!ret)
+ bch2_write_super(c);
+
mutex_unlock(&c->sb_lock);
+
return ret;
}
BUG_ON(ret < 0);
}
- percpu_u64_set(&c->usage[0]->replicas[idx], sectors);
+ c->usage_base->replicas[idx] = sectors;
return 0;
}
// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
+#include "buckets.h"
#include "checksum.h"
#include "disk_groups.h"
#include "ec.h"
mutex_unlock(&c->btree_root_lock);
- if (journal_seq)
- return entry;
+ percpu_down_read(&c->mark_lock);
- percpu_down_write(&c->mark_lock);
+ if (!journal_seq) {
+ for (i = 0; i < ARRAY_SIZE(c->usage); i++)
+ bch2_fs_usage_acc_to_base(c, i);
+ } else {
+ bch2_fs_usage_acc_to_base(c, journal_seq & 1);
+ }
{
- u64 nr_inodes = percpu_u64_get(&c->usage[0]->nr_inodes);
struct jset_entry_usage *u =
container_of(entry, struct jset_entry_usage, entry);
u->entry.u64s = DIV_ROUND_UP(sizeof(*u), sizeof(u64)) - 1;
u->entry.type = BCH_JSET_ENTRY_usage;
u->entry.btree_id = FS_USAGE_INODES;
- u->v = cpu_to_le64(nr_inodes);
+ u->v = cpu_to_le64(c->usage_base->nr_inodes);
entry = vstruct_next(entry);
}
for (i = 0; i < BCH_REPLICAS_MAX; i++) {
struct jset_entry_usage *u =
container_of(entry, struct jset_entry_usage, entry);
- u64 sectors = percpu_u64_get(&c->usage[0]->persistent_reserved[i]);
-
- if (!sectors)
- continue;
memset(u, 0, sizeof(*u));
u->entry.u64s = DIV_ROUND_UP(sizeof(*u), sizeof(u64)) - 1;
u->entry.type = BCH_JSET_ENTRY_usage;
u->entry.btree_id = FS_USAGE_RESERVED;
u->entry.level = i;
- u->v = sectors;
+ u->v = cpu_to_le64(c->usage_base->persistent_reserved[i]);
entry = vstruct_next(entry);
}
for (i = 0; i < c->replicas.nr; i++) {
struct bch_replicas_entry *e =
cpu_replicas_entry(&c->replicas, i);
- u64 sectors = percpu_u64_get(&c->usage[0]->replicas[i]);
struct jset_entry_data_usage *u =
container_of(entry, struct jset_entry_data_usage, entry);
u->entry.u64s = DIV_ROUND_UP(sizeof(*u) + e->nr_devs,
sizeof(u64)) - 1;
u->entry.type = BCH_JSET_ENTRY_data_usage;
- u->v = cpu_to_le64(sectors);
+ u->v = cpu_to_le64(c->usage_base->replicas[i]);
unsafe_memcpy(&u->r, e, replicas_entry_bytes(e),
"embedded variable length struct");
entry = vstruct_next(entry);
}
- percpu_up_write(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
return entry;
}
bch2_io_clock_exit(&c->io_clock[READ]);
bch2_fs_compress_exit(c);
percpu_free_rwsem(&c->mark_lock);
+ free_percpu(c->online_reserved);
kfree(c->usage_scratch);
+ free_percpu(c->usage[1]);
free_percpu(c->usage[0]);
+ kfree(c->usage_base);
free_percpu(c->pcpu);
mempool_exit(&c->btree_iters_pool);
mempool_exit(&c->btree_bounce_pool);
seqcount_init(&c->gc_pos_lock);
+ seqcount_init(&c->usage_lock);
+
c->copy_gc_enabled = 1;
c->rebalance.enabled = 1;
c->promote_whole_extents = true;
offsetof(struct btree_write_bio, wbio.bio)),
BIOSET_NEED_BVECS) ||
!(c->pcpu = alloc_percpu(struct bch_fs_pcpu)) ||
+ !(c->online_reserved = alloc_percpu(u64)) ||
mempool_init_kvpmalloc_pool(&c->btree_bounce_pool, 1,
btree_bytes(c)) ||
mempool_init_kmalloc_pool(&c->btree_iters_pool, 1,
static void dev_usage_clear(struct bch_dev *ca)
{
struct bucket_array *buckets;
- int cpu;
- for_each_possible_cpu(cpu) {
- struct bch_dev_usage *p =
- per_cpu_ptr(ca->usage[0], cpu);
- memset(p, 0, sizeof(*p));
- }
+ percpu_memset(ca->usage[0], 0, sizeof(*ca->usage[0]));
down_read(&ca->bucket_lock);
buckets = bucket_array(ca);
static ssize_t show_fs_alloc_debug(struct bch_fs *c, char *buf)
{
struct printbuf out = _PBUF(buf, PAGE_SIZE);
- struct bch_fs_usage *fs_usage = bch2_fs_usage_read(c);
- unsigned i;
+ struct bch_fs_usage_online *fs_usage = bch2_fs_usage_read(c);
if (!fs_usage)
return -ENOMEM;
- pr_buf(&out, "capacity:\t\t\t%llu\n", c->capacity);
-
- pr_buf(&out, "hidden:\t\t\t\t%llu\n",
- fs_usage->hidden);
- pr_buf(&out, "data:\t\t\t\t%llu\n",
- fs_usage->data);
- pr_buf(&out, "cached:\t\t\t\t%llu\n",
- fs_usage->cached);
- pr_buf(&out, "reserved:\t\t\t%llu\n",
- fs_usage->reserved);
- pr_buf(&out, "nr_inodes:\t\t\t%llu\n",
- fs_usage->nr_inodes);
- pr_buf(&out, "online reserved:\t\t%llu\n",
- fs_usage->online_reserved);
-
- for (i = 0;
- i < ARRAY_SIZE(fs_usage->persistent_reserved);
- i++) {
- pr_buf(&out, "%u replicas:\n", i + 1);
- pr_buf(&out, "\treserved:\t\t%llu\n",
- fs_usage->persistent_reserved[i]);
- }
-
- for (i = 0; i < c->replicas.nr; i++) {
- struct bch_replicas_entry *e =
- cpu_replicas_entry(&c->replicas, i);
-
- pr_buf(&out, "\t");
- bch2_replicas_entry_to_text(&out, e);
- pr_buf(&out, ":\t%llu\n", fs_usage->replicas[i]);
- }
+ bch2_fs_usage_to_text(&out, c, fs_usage);
percpu_up_read(&c->mark_lock);
"free[RESERVE_NONE]: %zu/%zu\n"
"buckets:\n"
" capacity: %llu\n"
- " alloc: %llu\n"
" sb: %llu\n"
" journal: %llu\n"
" meta: %llu\n"
fifo_used(&ca->free[RESERVE_MOVINGGC]), ca->free[RESERVE_MOVINGGC].size,
fifo_used(&ca->free[RESERVE_NONE]), ca->free[RESERVE_NONE].size,
ca->mi.nbuckets - ca->mi.first_bucket,
- stats.buckets_alloc,
stats.buckets[BCH_DATA_SB],
stats.buckets[BCH_DATA_JOURNAL],
stats.buckets[BCH_DATA_BTREE],
acc_u64s(acc, per_cpu_ptr(src, cpu), nr);
}
+static inline void percpu_memset(void __percpu *p, int c, size_t bytes)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ memset(per_cpu_ptr(p, cpu), c, bytes);
+}
+
u64 *bch2_acc_percpu_u64s(u64 __percpu *, unsigned);
#define cmp_int(l, r) ((l > r) - (l < r))
projects / linux.git / commitdiff