mm/slub: introduce __kmem_cache_free_bulk() without free hooks

Currently, when __kmem_cache_alloc_bulk() fails, it frees back the
objects that were allocated before the failure, using
kmem_cache_free_bulk(). Because kmem_cache_free_bulk() calls the free
hooks (KASAN etc.) and those expect objects that were processed by the
post alloc hooks, slab_post_alloc_hook() is called before
kmem_cache_free_bulk().

This is wasteful, although not a big concern in practice for the rare
error path. But in order to efficiently handle percpu array batch refill
and free in the near future, we will also need a variant of
kmem_cache_free_bulk() that avoids the free hooks. So introduce it now
and use it for the failure path.

In case of failure we however still need to perform memcg uncharge so
handle that in a new memcg_slab_alloc_error_hook(). Thanks to Chengming
Zhou for noticing the missing uncharge.

As a consequence, __kmem_cache_alloc_bulk() no longer needs the objcg
parameter, remove it.

Reviewed-by: Chengming Zhou <zhouchengming@bytedance.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/mm/slub.c b/mm/slub.c
index d7b0ca6012e02a1c60d1eecd493ba293a0c524ca..0a9e4bd0dd6857100519fe5963098e0fa5a47d77 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2003,6 +2003,14 @@ void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
 
        __memcg_slab_free_hook(s, slab, p, objects, objcgs);
 }
+
+static inline
+void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects,
+                          struct obj_cgroup *objcg)
+{
+       if (objcg)
+               obj_cgroup_uncharge(objcg, objects * obj_full_size(s));
+}
 #else /* CONFIG_MEMCG_KMEM */
 static inline struct mem_cgroup *memcg_from_slab_obj(void *ptr)
 {
@@ -2032,6 +2040,12 @@ static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
                                        void **p, int objects)
 {
 }
+
+static inline
+void memcg_slab_alloc_error_hook(struct kmem_cache *s, int objects,
+                                struct obj_cgroup *objcg)
+{
+}
 #endif /* CONFIG_MEMCG_KMEM */
 
 /*
@@ -4478,6 +4492,27 @@ int build_detached_freelist(struct kmem_cache *s, size_t size,
        return same;
 }
 
+/*
+ * Internal bulk free of objects that were not initialised by the post alloc
+ * hooks and thus should not be processed by the free hooks
+ */
+static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
+{
+       if (!size)
+               return;
+
+       do {
+               struct detached_freelist df;
+
+               size = build_detached_freelist(s, size, p, &df);
+               if (!df.slab)
+                       continue;
+
+               do_slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt,
+                            _RET_IP_);
+       } while (likely(size));
+}
+
 /* Note that interrupts must be enabled when calling this function. */
 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
 {
@@ -4498,8 +4533,9 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
 EXPORT_SYMBOL(kmem_cache_free_bulk);
 
 #ifndef CONFIG_SLUB_TINY
-static inline int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
-                       size_t size, void **p, struct obj_cgroup *objcg)
+static inline
+int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
+                           void **p)
 {
        struct kmem_cache_cpu *c;
        unsigned long irqflags;
@@ -4563,14 +4599,13 @@ static inline int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
 
 error:
        slub_put_cpu_ptr(s->cpu_slab);
-       slab_post_alloc_hook(s, objcg, flags, i, p, false, s->object_size);
-       kmem_cache_free_bulk(s, i, p);
+       __kmem_cache_free_bulk(s, i, p);
        return 0;
 
 }
 #else /* CONFIG_SLUB_TINY */
 static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
-                       size_t size, void **p, struct obj_cgroup *objcg)
+                                  size_t size, void **p)
 {
        int i;
 
@@ -4593,8 +4628,7 @@ static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
        return i;
 
 error:
-       slab_post_alloc_hook(s, objcg, flags, i, p, false, s->object_size);
-       kmem_cache_free_bulk(s, i, p);
+       __kmem_cache_free_bulk(s, i, p);
        return 0;
 }
 #endif /* CONFIG_SLUB_TINY */
@@ -4614,15 +4648,19 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
        if (unlikely(!s))
                return 0;
 
-       i = __kmem_cache_alloc_bulk(s, flags, size, p, objcg);
+       i = __kmem_cache_alloc_bulk(s, flags, size, p);
 
        /*
         * memcg and kmem_cache debug support and memory initialization.
         * Done outside of the IRQ disabled fastpath loop.
         */
-       if (i != 0)
+       if (likely(i != 0)) {
                slab_post_alloc_hook(s, objcg, flags, size, p,
                        slab_want_init_on_alloc(flags, s), s->object_size);
+       } else {
+               memcg_slab_alloc_error_hook(s, size, objcg);
+       }
+
        return i;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_bulk);