bcachefs: Inline fast path of bch2_increment_clock()

Shaving more cycles.

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/clock.c b/fs/bcachefs/clock.c
index e4486fcbea193b88d70661ff62b0fdc252cb8396..e227753563abd26498d4e3338f273840310852d6 100644 (file)
--- a/fs/bcachefs/clock.c
+++ b/fs/bcachefs/clock.c
@@ -135,17 +135,16 @@ static struct io_timer *get_expired_timer(struct io_clock *clock,
        return ret;
 }
 
-void bch2_increment_clock(struct bch_fs *c, unsigned sectors, int rw)
+void __bch2_increment_clock(struct io_clock *clock)
 {
-       struct io_clock *clock = &c->io_clock[rw];
        struct io_timer *timer;
        unsigned long now;
+       unsigned sectors;
 
        /* Buffer up one megabyte worth of IO in the percpu counter */
        preempt_disable();
 
-       if (likely(this_cpu_add_return(*clock->pcpu_buf, sectors) <
-                  IO_CLOCK_PCPU_SECTORS)) {
+       if (this_cpu_read(*clock->pcpu_buf) < IO_CLOCK_PCPU_SECTORS) {
                preempt_enable();
                return;
        }

diff --git a/fs/bcachefs/clock.h b/fs/bcachefs/clock.h
index 5cb043c579d8b6a68b70f23496e7a450a6829165..bfbbca8a207b0b8a3621d443bf9d99dfeb209c1d 100644 (file)
--- a/fs/bcachefs/clock.h
+++ b/fs/bcachefs/clock.h
@@ -6,7 +6,18 @@ void bch2_io_timer_add(struct io_clock *, struct io_timer *);
 void bch2_io_timer_del(struct io_clock *, struct io_timer *);
 void bch2_kthread_io_clock_wait(struct io_clock *, unsigned long,
                                unsigned long);
-void bch2_increment_clock(struct bch_fs *, unsigned, int);
+
+void __bch2_increment_clock(struct io_clock *);
+
+static inline void bch2_increment_clock(struct bch_fs *c, unsigned sectors,
+                                       int rw)
+{
+       struct io_clock *clock = &c->io_clock[rw];
+
+       if (unlikely(this_cpu_add_return(*clock->pcpu_buf, sectors) >=
+                  IO_CLOCK_PCPU_SECTORS))
+               __bch2_increment_clock(clock);
+}
 
 void bch2_io_clock_schedule_timeout(struct io_clock *, unsigned long);