bcachefs: Inline fastpath of bch2_disk_reservation_add()

The fastpath now doesn't even disable preemption - instead we use a (non
locked) cmpxchg.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c
index b37cdf7279de6faae587bf9e52cc1a0111966bfc..6389ec7ba18b065a408ddade4c78a2d91a5eab4d 100644 (file)
--- a/fs/bcachefs/buckets.c
+++ b/fs/bcachefs/buckets.c
@@ -2107,7 +2107,7 @@ int bch2_trans_mark_dev_sb(struct bch_fs *c, struct bch_dev *ca)
 
 #define SECTORS_CACHE  1024
 
-int bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res,
+int __bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res,
                              u64 sectors, int flags)
 {
        struct bch_fs_pcpu *pcpu;

diff --git a/fs/bcachefs/buckets.h b/fs/bcachefs/buckets.h
index 04a2a9310cdd9e9fc4d4ceb7a003d618540d7205..61be96a7b03d14a8087e0fa5a373ddd0f56539c1 100644 (file)
--- a/fs/bcachefs/buckets.h
+++ b/fs/bcachefs/buckets.h
@@ -261,15 +261,35 @@ int bch2_trans_mark_dev_sb(struct bch_fs *, struct bch_dev *);
 static inline void bch2_disk_reservation_put(struct bch_fs *c,
                                             struct disk_reservation *res)
 {
-       this_cpu_sub(*c->online_reserved, res->sectors);
-       res->sectors = 0;
+       if (res->sectors) {
+               this_cpu_sub(*c->online_reserved, res->sectors);
+               res->sectors = 0;
+       }
 }
 
 #define BCH_DISK_RESERVATION_NOFAIL            (1 << 0)
 
-int bch2_disk_reservation_add(struct bch_fs *,
-                             struct disk_reservation *,
-                             u64, int);
+int __bch2_disk_reservation_add(struct bch_fs *,
+                               struct disk_reservation *,
+                               u64, int);
+
+static inline int bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res,
+                                           u64 sectors, int flags)
+{
+       u64 old, new;
+
+       do {
+               old = this_cpu_read(c->pcpu->sectors_available);
+               if (sectors > old)
+                       return __bch2_disk_reservation_add(c, res, sectors, flags);
+
+               new = old - sectors;
+       } while (this_cpu_cmpxchg(c->pcpu->sectors_available, old, new) != old);
+
+       this_cpu_add(*c->online_reserved, sectors);
+       res->sectors                    += sectors;
+       return 0;
+}
 
 static inline struct disk_reservation
 bch2_disk_reservation_init(struct bch_fs *c, unsigned nr_replicas)