mm/slub: refactor deactivate_slab()

Simplify deactivate_slab() by unlocking n->list_lock and retrying
cmpxchg_double() when cmpxchg_double() fails, and perform
add_{partial,full} only when it succeed.

Releasing and taking n->list_lock again here is not harmful as SLUB
avoids deactivating slabs as much as possible.

[ vbabka@suse.cz: perform add_{partial,full} when cmpxchg_double()
  succeed.

  count deactivating full slabs even if debugging flag is not set. ]

Signed-off-by: Hyeonggon Yoo <42.hyeyoo@gmail.com>
Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
Reviewed-by: Roman Gushchin <roman.gushchin@linux.dev>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Link: https://lore.kernel.org/r/20220307074057.902222-3-42.hyeyoo@gmail.com

diff --git a/mm/slub.c b/mm/slub.c
index 1ce09b0347ad5bfb99c0f500b2ff45578384c387..e6df45e12b86b0acfdd80bf69508fd8fdb70d8d2 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2348,10 +2348,10 @@ static void init_kmem_cache_cpus(struct kmem_cache *s)
 static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
                            void *freelist)
 {
-       enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
+       enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE, M_FULL_NOLIST };
        struct kmem_cache_node *n = get_node(s, slab_nid(slab));
-       int lock = 0, free_delta = 0;
-       enum slab_modes l = M_NONE, m = M_NONE;
+       int free_delta = 0;
+       enum slab_modes mode = M_NONE;
        void *nextfree, *freelist_iter, *freelist_tail;
        int tail = DEACTIVATE_TO_HEAD;
        unsigned long flags = 0;
@@ -2393,14 +2393,10 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
         * Ensure that the slab is unfrozen while the list presence
         * reflects the actual number of objects during unfreeze.
         *
-        * We setup the list membership and then perform a cmpxchg
-        * with the count. If there is a mismatch then the slab
-        * is not unfrozen but the slab is on the wrong list.
-        *
-        * Then we restart the process which may have to remove
-        * the slab from the list that we just put it on again
-        * because the number of objects in the slab may have
-        * changed.
+        * We first perform cmpxchg holding lock and insert to list
+        * when it succeed. If there is mismatch then the slab is not
+        * unfrozen and number of objects in the slab may have changed.
+        * Then release lock and retry cmpxchg again.
         */
 redo:
 
@@ -2419,61 +2415,52 @@ redo:
 
        new.frozen = 0;
 
-       if (!new.inuse && n->nr_partial >= s->min_partial)
-               m = M_FREE;
-       else if (new.freelist) {
-               m = M_PARTIAL;
-               if (!lock) {
-                       lock = 1;
-                       /*
-                        * Taking the spinlock removes the possibility that
-                        * acquire_slab() will see a slab that is frozen
-                        */
-                       spin_lock_irqsave(&n->list_lock, flags);
-               }
+       if (!new.inuse && n->nr_partial >= s->min_partial) {
+               mode = M_FREE;
+       } else if (new.freelist) {
+               mode = M_PARTIAL;
+               /*
+                * Taking the spinlock removes the possibility that
+                * acquire_slab() will see a slab that is frozen
+                */
+               spin_lock_irqsave(&n->list_lock, flags);
+       } else if (kmem_cache_debug_flags(s, SLAB_STORE_USER)) {
+               mode = M_FULL;
+               /*
+                * This also ensures that the scanning of full
+                * slabs from diagnostic functions will not see
+                * any frozen slabs.
+                */
+               spin_lock_irqsave(&n->list_lock, flags);
        } else {
-               m = M_FULL;
-               if (kmem_cache_debug_flags(s, SLAB_STORE_USER) && !lock) {
-                       lock = 1;
-                       /*
-                        * This also ensures that the scanning of full
-                        * slabs from diagnostic functions will not see
-                        * any frozen slabs.
-                        */
-                       spin_lock_irqsave(&n->list_lock, flags);
-               }
+               mode = M_FULL_NOLIST;
        }
 
-       if (l != m) {
-               if (l == M_PARTIAL)
-                       remove_partial(n, slab);
-               else if (l == M_FULL)
-                       remove_full(s, n, slab);
 
-               if (m == M_PARTIAL)
-                       add_partial(n, slab, tail);
-               else if (m == M_FULL)
-                       add_full(s, n, slab);
-       }
-
-       l = m;
        if (!cmpxchg_double_slab(s, slab,
                                old.freelist, old.counters,
                                new.freelist, new.counters,
-                               "unfreezing slab"))
+                               "unfreezing slab")) {
+               if (mode == M_PARTIAL || mode == M_FULL)
+                       spin_unlock_irqrestore(&n->list_lock, flags);
                goto redo;
+       }
 
-       if (lock)
-               spin_unlock_irqrestore(&n->list_lock, flags);
 
-       if (m == M_PARTIAL)
+       if (mode == M_PARTIAL) {
+               add_partial(n, slab, tail);
+               spin_unlock_irqrestore(&n->list_lock, flags);
                stat(s, tail);
-       else if (m == M_FULL)
-               stat(s, DEACTIVATE_FULL);
-       else if (m == M_FREE) {
+       } else if (mode == M_FREE) {
                stat(s, DEACTIVATE_EMPTY);
                discard_slab(s, slab);
                stat(s, FREE_SLAB);
+       } else if (mode == M_FULL) {
+               add_full(s, n, slab);
+               spin_unlock_irqrestore(&n->list_lock, flags);
+               stat(s, DEACTIVATE_FULL);
+       } else if (mode == M_FULL_NOLIST) {
+               stat(s, DEACTIVATE_FULL);
        }
 }