bcachefs: Better inlining in bch2_time_stats_update()

Move the actual slowpath off into a new function -
bch2_time_stats_clear_buffer() - and inline
bch2_time_stats_update_one().

Alo, use the new inlined update functions from mean_and_variance.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/fs/bcachefs/util.c b/fs/bcachefs/util.c
index 8b2eef24498e9ac59d537ff4f6f36f257a072ec2..31934f7a6436a5ada1746215b68d7c4a1bc819a1 100644 (file)
--- a/fs/bcachefs/util.c
+++ b/fs/bcachefs/util.c
@@ -319,8 +319,8 @@ static void bch2_quantiles_update(struct bch2_quantiles *q, u64 v)
        }
 }
 
-static void bch2_time_stats_update_one(struct bch2_time_stats *stats,
-                                      u64 start, u64 end)
+static inline void bch2_time_stats_update_one(struct bch2_time_stats *stats,
+                                             u64 start, u64 end)
 {
        u64 duration, freq;
 
@@ -343,6 +343,22 @@ static void bch2_time_stats_update_one(struct bch2_time_stats *stats,
        }
 }
 
+static noinline void bch2_time_stats_clear_buffer(struct bch2_time_stats *stats,
+                                                 struct bch2_time_stat_buffer *b)
+{
+       struct bch2_time_stat_buffer_entry *i;
+       unsigned long flags;
+
+       spin_lock_irqsave(&stats->lock, flags);
+       for (i = b->entries;
+            i < b->entries + ARRAY_SIZE(b->entries);
+            i++)
+               bch2_time_stats_update_one(stats, i->start, i->end);
+       spin_unlock_irqrestore(&stats->lock, flags);
+
+       b->nr = 0;
+}
+
 void __bch2_time_stats_update(struct bch2_time_stats *stats, u64 start, u64 end)
 {
        unsigned long flags;
@@ -362,7 +378,6 @@ void __bch2_time_stats_update(struct bch2_time_stats *stats, u64 start, u64 end)
                                                 GFP_ATOMIC);
                spin_unlock_irqrestore(&stats->lock, flags);
        } else {
-               struct bch2_time_stat_buffer_entry *i;
                struct bch2_time_stat_buffer *b;
 
                preempt_disable();
@@ -374,17 +389,8 @@ void __bch2_time_stats_update(struct bch2_time_stats *stats, u64 start, u64 end)
                        .end = end
                };
 
-               if (b->nr == ARRAY_SIZE(b->entries)) {
-                       spin_lock_irqsave(&stats->lock, flags);
-                       for (i = b->entries;
-                            i < b->entries + ARRAY_SIZE(b->entries);
-                            i++)
-                               bch2_time_stats_update_one(stats, i->start, i->end);
-                       spin_unlock_irqrestore(&stats->lock, flags);
-
-                       b->nr = 0;
-               }
-
+               if (unlikely(b->nr == ARRAY_SIZE(b->entries)))
+                       bch2_time_stats_clear_buffer(stats, b);
                preempt_enable();
        }
 }