fsnotify: optimize the case of no permission event watchers

Commit e43de7f0862b ("fsnotify: optimize the case of no marks of any type")
optimized the case where there are no fsnotify watchers on any of the
filesystem's objects.

It is quite common for a system to have a single local filesystem and
it is quite common for the system to have some inotify watches on some
config files or directories, so the optimization of no marks at all is
often not in effect.

Permission event watchers, which require high priority group are more
rare, so optimizing the case of no marks og high priority groups can
improve performance for more systems, especially for performance
sensitive io workloads.

Count per-sb watched objects by high priority groups and use that the
optimize out the call to __fsnotify_parent() and fsnotify() in fsnotify
permission hooks.

Signed-off-by: Amir Goldstein <amir73il@gmail.com>
Signed-off-by: Jan Kara <jack@suse.cz>
Message-Id: <20240317184154.1200192-11-amir73il@gmail.com>

diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c
index fb3f36bc6ea9e43ef12263caaaadd79c39b36135..2ae965ef37e859976a997a9d0329e2094c03002b 100644 (file)
--- a/fs/notify/fsnotify.c
+++ b/fs/notify/fsnotify.c
@@ -100,6 +100,9 @@ void fsnotify_sb_delete(struct super_block *sb)
        /* Wait for outstanding object references from connectors */
        wait_var_event(fsnotify_sb_watched_objects(sb),
                       !atomic_long_read(fsnotify_sb_watched_objects(sb)));
+       WARN_ON(fsnotify_sb_has_priority_watchers(sb, FSNOTIFY_PRIO_CONTENT));
+       WARN_ON(fsnotify_sb_has_priority_watchers(sb,
+                                                 FSNOTIFY_PRIO_PRE_CONTENT));
        kfree(sbinfo);
 }
 

diff --git a/fs/notify/mark.c b/fs/notify/mark.c
index b2f5d8c9cce11546be7e008b9ffba206ae28821f..c3eefa70633c4deadbbeb38136ed771aa7fd90cc 100644 (file)
--- a/fs/notify/mark.c
+++ b/fs/notify/mark.c
@@ -161,13 +161,36 @@ static void fsnotify_put_inode_ref(struct inode *inode)
 static void fsnotify_update_sb_watchers(struct super_block *sb,
                                        struct fsnotify_mark_connector *conn)
 {
+       struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb);
        bool is_watched = conn->flags & FSNOTIFY_CONN_FLAG_IS_WATCHED;
-       bool has_marks = conn->obj && !hlist_empty(&conn->list);
+       struct fsnotify_mark *first_mark = NULL;
+       unsigned int highest_prio = 0;
 
-       if (has_marks && !is_watched) {
+       if (conn->obj)
+               first_mark = hlist_entry_safe(conn->list.first,
+                                             struct fsnotify_mark, obj_list);
+       if (first_mark)
+               highest_prio = first_mark->group->priority;
+       if (WARN_ON(highest_prio >= __FSNOTIFY_PRIO_NUM))
+               highest_prio = 0;
+
+       /*
+        * If the highest priority of group watching this object is prio,
+        * then watched object has a reference on counters [0..prio].
+        * Update priority >= 1 watched objects counters.
+        */
+       for (unsigned int p = conn->prio + 1; p <= highest_prio; p++)
+               atomic_long_inc(&sbinfo->watched_objects[p]);
+       for (unsigned int p = conn->prio; p > highest_prio; p--)
+               atomic_long_dec(&sbinfo->watched_objects[p]);
+       conn->prio = highest_prio;
+
+       /* Update priority >= 0 (a.k.a total) watched objects counter */
+       BUILD_BUG_ON(FSNOTIFY_PRIO_NORMAL != 0);
+       if (first_mark && !is_watched) {
                conn->flags |= FSNOTIFY_CONN_FLAG_IS_WATCHED;
                fsnotify_get_sb_watched_objects(sb);
-       } else if (!has_marks && is_watched) {
+       } else if (!first_mark && is_watched) {
                conn->flags &= ~FSNOTIFY_CONN_FLAG_IS_WATCHED;
                fsnotify_put_sb_watched_objects(sb);
        }
@@ -600,6 +623,7 @@ static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp,
        spin_lock_init(&conn->lock);
        INIT_HLIST_HEAD(&conn->list);
        conn->flags = 0;
+       conn->prio = 0;
        conn->type = obj_type;
        conn->obj = obj;
 

diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h
index 48dc657024151fadb24fef9b883b8c70191f42ec..4da80e92f804f931f82e01ec8f8e5b959c76097c 100644 (file)
--- a/include/linux/fsnotify.h
+++ b/include/linux/fsnotify.h
@@ -17,8 +17,9 @@
 #include <linux/slab.h>
 #include <linux/bug.h>
 
-/* Are there any inode/mount/sb objects that are being watched at all? */
-static inline bool fsnotify_sb_has_watchers(struct super_block *sb)
+/* Are there any inode/mount/sb objects watched with priority prio or above? */
+static inline bool fsnotify_sb_has_priority_watchers(struct super_block *sb,
+                                                    int prio)
 {
        struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb);
 
@@ -26,7 +27,13 @@ static inline bool fsnotify_sb_has_watchers(struct super_block *sb)
        if (!sbinfo)
                return false;
 
-       return atomic_long_read(&sbinfo->watched_objects);
+       return atomic_long_read(&sbinfo->watched_objects[prio]);
+}
+
+/* Are there any inode/mount/sb objects that are being watched at all? */
+static inline bool fsnotify_sb_has_watchers(struct super_block *sb)
+{
+       return fsnotify_sb_has_priority_watchers(sb, 0);
 }
 
 /*
@@ -109,6 +116,12 @@ static inline int fsnotify_file(struct file *file, __u32 mask)
                return 0;
 
        path = &file->f_path;
+       /* Permission events require group prio >= FSNOTIFY_PRIO_CONTENT */
+       if (mask & ALL_FSNOTIFY_PERM_EVENTS &&
+           !fsnotify_sb_has_priority_watchers(path->dentry->d_sb,
+                                              FSNOTIFY_PRIO_CONTENT))
+               return 0;
+
        return fsnotify_parent(path->dentry, mask, path, FSNOTIFY_EVENT_PATH);
 }
 

diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h
index fc38587d85643af08bd961c1a2c8d4fe3d86c714..7f1ab8264e41d3961aff09befec6c2fce46b1f5f 100644 (file)
--- a/include/linux/fsnotify_backend.h
+++ b/include/linux/fsnotify_backend.h
@@ -468,7 +468,8 @@ FSNOTIFY_ITER_FUNCS(sb, SB)
  */
 struct fsnotify_mark_connector {
        spinlock_t lock;
-       unsigned short type;    /* Type of object [lock] */
+       unsigned char type;     /* Type of object [lock] */
+       unsigned char prio;     /* Highest priority group */
 #define FSNOTIFY_CONN_FLAG_IS_WATCHED  0x01
 #define FSNOTIFY_CONN_FLAG_HAS_IREF    0x02
        unsigned short flags;   /* flags [lock] */
@@ -490,8 +491,12 @@ struct fsnotify_sb_info {
        /*
         * Number of inode/mount/sb objects that are being watched in this sb.
         * Note that inodes objects are currently double-accounted.
+        *
+        * The value in watched_objects[prio] is the number of objects that are
+        * watched by groups of priority >= prio, so watched_objects[0] is the
+        * total number of watched objects in this sb.
         */
-       atomic_long_t watched_objects;
+       atomic_long_t watched_objects[__FSNOTIFY_PRIO_NUM];
 };
 
 static inline struct fsnotify_sb_info *fsnotify_sb_info(struct super_block *sb)
@@ -505,7 +510,7 @@ static inline struct fsnotify_sb_info *fsnotify_sb_info(struct super_block *sb)
 
 static inline atomic_long_t *fsnotify_sb_watched_objects(struct super_block *sb)
 {
-       return &fsnotify_sb_info(sb)->watched_objects;
+       return &fsnotify_sb_info(sb)->watched_objects[0];
 }
 
 /*