ceph: metrics for opened files, pinned caps and opened inodes

In client for each inode, it may have many opened files and may
have been pinned in more than one MDS servers. And some inodes
are idle, which have no any opened files.

This patch will show these metrics in the debugfs, likes:

item                               total
-----------------------------------------
opened files  / total inodes       14 / 5
pinned i_caps / total inodes       7  / 5
opened inodes / total inodes       3  / 5

Will send these metrics to ceph, which will be used by the `fs top`,
later.

[ jlayton: drop unrelated hunk, count hashed inodes instead of
           allocated ones ]

URL: https://tracker.ceph.com/issues/47005
Signed-off-by: Xiubo Li <xiubli@redhat.com>
Signed-off-by: Jeff Layton <jlayton@kernel.org>
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>

diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c
index 39753e0e0e5a288d8d3e7555ff2ce25d42199e0f..c7e69547628e703b564969b10a0a4beb83f99301 100644 (file)
--- a/fs/ceph/caps.c
+++ b/fs/ceph/caps.c
@@ -4283,13 +4283,30 @@ void __ceph_touch_fmode(struct ceph_inode_info *ci,
 
 void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count)
 {
-       int i;
+       struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(ci->vfs_inode.i_sb);
        int bits = (fmode << 1) | 1;
+       bool is_opened = false;
+       int i;
+
+       if (count == 1)
+               atomic64_inc(&mdsc->metric.opened_files);
+
        spin_lock(&ci->i_ceph_lock);
        for (i = 0; i < CEPH_FILE_MODE_BITS; i++) {
                if (bits & (1 << i))
                        ci->i_nr_by_mode[i] += count;
+
+               /*
+                * If any of the mode ref is larger than 1,
+                * that means it has been already opened by
+                * others. Just skip checking the PIN ref.
+                */
+               if (i && ci->i_nr_by_mode[i] > 1)
+                       is_opened = true;
        }
+
+       if (!is_opened)
+               percpu_counter_inc(&mdsc->metric.opened_inodes);
        spin_unlock(&ci->i_ceph_lock);
 }
 
@@ -4300,15 +4317,32 @@ void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count)
  */
 void ceph_put_fmode(struct ceph_inode_info *ci, int fmode, int count)
 {
-       int i;
+       struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(ci->vfs_inode.i_sb);
        int bits = (fmode << 1) | 1;
+       bool is_closed = true;
+       int i;
+
+       if (count == 1)
+               atomic64_dec(&mdsc->metric.opened_files);
+
        spin_lock(&ci->i_ceph_lock);
        for (i = 0; i < CEPH_FILE_MODE_BITS; i++) {
                if (bits & (1 << i)) {
                        BUG_ON(ci->i_nr_by_mode[i] < count);
                        ci->i_nr_by_mode[i] -= count;
                }
+
+               /*
+                * If any of the mode ref is not 0 after
+                * decreased, that means it is still opened
+                * by others. Just skip checking the PIN ref.
+                */
+               if (i && ci->i_nr_by_mode[i])
+                       is_closed = false;
        }
+
+       if (is_closed)
+               percpu_counter_dec(&mdsc->metric.opened_inodes);
        spin_unlock(&ci->i_ceph_lock);
 }
 

diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c
index 75b13175c530553322d7ac536f9ab5e28f5a429e..7a8fbe3e47514868136ce78d651de41da6750008 100644 (file)
--- a/fs/ceph/debugfs.c
+++ b/fs/ceph/debugfs.c
@@ -148,6 +148,17 @@ static int metric_show(struct seq_file *s, void *p)
        int nr_caps = 0;
        s64 total, sum, avg, min, max, sq;
 
+       sum = percpu_counter_sum(&m->total_inodes);
+       seq_printf(s, "item                               total\n");
+       seq_printf(s, "------------------------------------------\n");
+       seq_printf(s, "%-35s%lld / %lld\n", "opened files  / total inodes",
+                  atomic64_read(&m->opened_files), sum);
+       seq_printf(s, "%-35s%lld / %lld\n", "pinned i_caps / total inodes",
+                  atomic64_read(&m->total_caps), sum);
+       seq_printf(s, "%-35s%lld / %lld\n", "opened inodes / total inodes",
+                  percpu_counter_sum(&m->opened_inodes), sum);
+
+       seq_printf(s, "\n");
        seq_printf(s, "item          total       avg_lat(us)     min_lat(us)     max_lat(us)     stdev(us)\n");
        seq_printf(s, "-----------------------------------------------------------------------------------\n");
 

diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c
index 1fed0e827eb79766cf396d829aeef19a3f435bc7..526faf4778ce43f60adf61680caef34fa2852f29 100644 (file)
--- a/fs/ceph/inode.c
+++ b/fs/ceph/inode.c
@@ -42,10 +42,13 @@ static void ceph_inode_work(struct work_struct *work);
 static int ceph_set_ino_cb(struct inode *inode, void *data)
 {
        struct ceph_inode_info *ci = ceph_inode(inode);
+       struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
 
        ci->i_vino = *(struct ceph_vino *)data;
        inode->i_ino = ceph_vino_to_ino_t(ci->i_vino);
        inode_set_iversion_raw(inode, 0);
+       percpu_counter_inc(&mdsc->metric.total_inodes);
+
        return 0;
 }
 
@@ -538,11 +541,14 @@ void ceph_free_inode(struct inode *inode)
 void ceph_evict_inode(struct inode *inode)
 {
        struct ceph_inode_info *ci = ceph_inode(inode);
+       struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
        struct ceph_inode_frag *frag;
        struct rb_node *n;
 
        dout("evict_inode %p ino %llx.%llx\n", inode, ceph_vinop(inode));
 
+       percpu_counter_dec(&mdsc->metric.total_inodes);
+
        truncate_inode_pages_final(&inode->i_data);
        clear_inode(inode);
 
@@ -558,7 +564,6 @@ void ceph_evict_inode(struct inode *inode)
         * caps in i_snap_caps.
         */
        if (ci->i_snap_realm) {
-               struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
                if (ceph_snap(inode) == CEPH_NOSNAP) {
                        struct ceph_snap_realm *realm = ci->i_snap_realm;
                        dout(" dropping residual ref to snap realm %p\n",

diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c
index 2466b261fba243a24cc0c2222e9e911ccb54b102..fee4c47783132deb1b4d528d199dea1dfd459e9c 100644 (file)
--- a/fs/ceph/metric.c
+++ b/fs/ceph/metric.c
@@ -192,11 +192,23 @@ int ceph_metric_init(struct ceph_client_metric *m)
        m->total_metadatas = 0;
        m->metadata_latency_sum = 0;
 
+       atomic64_set(&m->opened_files, 0);
+       ret = percpu_counter_init(&m->opened_inodes, 0, GFP_KERNEL);
+       if (ret)
+               goto err_opened_inodes;
+       ret = percpu_counter_init(&m->total_inodes, 0, GFP_KERNEL);
+       if (ret)
+               goto err_total_inodes;
+
        m->session = NULL;
        INIT_DELAYED_WORK(&m->delayed_work, metric_delayed_work);
 
        return 0;
 
+err_total_inodes:
+       percpu_counter_destroy(&m->opened_inodes);
+err_opened_inodes:
+       percpu_counter_destroy(&m->i_caps_mis);
 err_i_caps_mis:
        percpu_counter_destroy(&m->i_caps_hit);
 err_i_caps_hit:
@@ -212,6 +224,8 @@ void ceph_metric_destroy(struct ceph_client_metric *m)
        if (!m)
                return;
 
+       percpu_counter_destroy(&m->total_inodes);
+       percpu_counter_destroy(&m->opened_inodes);
        percpu_counter_destroy(&m->i_caps_mis);
        percpu_counter_destroy(&m->i_caps_hit);
        percpu_counter_destroy(&m->d_lease_mis);

diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h
index 1d0959d669d7023a90d25b931332d48f12180688..710f3f1dceab97388ea46adf0cdb4f8f357651e3 100644 (file)
--- a/fs/ceph/metric.h
+++ b/fs/ceph/metric.h
@@ -115,6 +115,13 @@ struct ceph_client_metric {
        ktime_t metadata_latency_min;
        ktime_t metadata_latency_max;
 
+       /* The total number of directories and files that are opened */
+       atomic64_t opened_files;
+
+       /* The total number of inodes that have opened files or directories */
+       struct percpu_counter opened_inodes;
+       struct percpu_counter total_inodes;
+
        struct ceph_mds_session *session;
        struct delayed_work delayed_work;  /* delayed work */
 };