mm: add free_unref_folios()

Iterate over a folio_batch rather than a linked list.  This is easier for
the CPU to prefetch and has a batch count naturally built in so we don't
need to track it.  Again, this lowers the maximum lock hold time from
32 folios to 15, but I do not expect this to have a significant effect.

Link: https://lkml.kernel.org/r/20240227174254.710559-4-willy@infradead.org
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/internal.h b/mm/internal.h
index f376e3afbc4c72239dd4bb36fffee9de74886bee..1dfdc3bde1b0199a46c27bf7e94e83ec3399c201 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -451,8 +451,9 @@ extern bool free_pages_prepare(struct page *page, unsigned int order);
 
 extern int user_min_free_kbytes;
 
-extern void free_unref_page(struct page *page, unsigned int order);
-extern void free_unref_page_list(struct list_head *list);
+void free_unref_page(struct page *page, unsigned int order);
+void free_unref_folios(struct folio_batch *fbatch);
+void free_unref_page_list(struct list_head *list);
 
 extern void zone_pcp_reset(struct zone *zone);
 extern void zone_pcp_disable(struct zone *zone);

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 20d4ba095ad251a8075816090e2bf6fd201ded43..31d97322feea0392cdf40a4f695249fd4be97f6a 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -32,6 +32,7 @@
 #include <linux/sysctl.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
+#include <linux/pagevec.h>
 #include <linux/memory_hotplug.h>
 #include <linux/nodemask.h>
 #include <linux/vmstat.h>
@@ -2515,57 +2516,51 @@ void free_unref_page(struct page *page, unsigned int order)
 }
 
 /*
- * Free a list of 0-order pages
+ * Free a batch of 0-order pages
  */
-void free_unref_page_list(struct list_head *list)
+void free_unref_folios(struct folio_batch *folios)
 {
        unsigned long __maybe_unused UP_flags;
-       struct folio *folio, *next;
        struct per_cpu_pages *pcp = NULL;
        struct zone *locked_zone = NULL;
-       int batch_count = 0;
-       int migratetype;
+       int i, j, migratetype;
 
-       /* Prepare pages for freeing */
-       list_for_each_entry_safe(folio, next, list, lru) {
+       /* Prepare folios for freeing */
+       for (i = 0, j = 0; i < folios->nr; i++) {
+               struct folio *folio = folios->folios[i];
                unsigned long pfn = folio_pfn(folio);
-               if (!free_unref_page_prepare(&folio->page, pfn, 0)) {
-                       list_del(&folio->lru);
+               if (!free_unref_page_prepare(&folio->page, pfn, 0))
                        continue;
-               }
 
                /*
-                * Free isolated pages directly to the allocator, see
+                * Free isolated folios directly to the allocator, see
                 * comment in free_unref_page.
                 */
                migratetype = get_pcppage_migratetype(&folio->page);
                if (unlikely(is_migrate_isolate(migratetype))) {
-                       list_del(&folio->lru);
                        free_one_page(folio_zone(folio), &folio->page, pfn,
                                        0, migratetype, FPI_NONE);
                        continue;
                }
+               if (j != i)
+                       folios->folios[j] = folio;
+               j++;
        }
+       folios->nr = j;
 
-       list_for_each_entry_safe(folio, next, list, lru) {
+       for (i = 0; i < folios->nr; i++) {
+               struct folio *folio = folios->folios[i];
                struct zone *zone = folio_zone(folio);
 
-               list_del(&folio->lru);
                migratetype = get_pcppage_migratetype(&folio->page);
 
-               /*
-                * Either different zone requiring a different pcp lock or
-                * excessive lock hold times when freeing a large list of
-                * folios.
-                */
-               if (zone != locked_zone || batch_count == SWAP_CLUSTER_MAX) {
+               /* Different zone requires a different pcp lock */
+               if (zone != locked_zone) {
                        if (pcp) {
                                pcp_spin_unlock(pcp);
                                pcp_trylock_finish(UP_flags);
                        }
 
-                       batch_count = 0;
-
                        /*
                         * trylock is necessary as folios may be getting freed
                         * from IRQ or SoftIRQ context after an IO completion.
@@ -2592,13 +2587,31 @@ void free_unref_page_list(struct list_head *list)
 
                trace_mm_page_free_batched(&folio->page);
                free_unref_page_commit(zone, pcp, &folio->page, migratetype, 0);
-               batch_count++;
        }
 
        if (pcp) {
                pcp_spin_unlock(pcp);
                pcp_trylock_finish(UP_flags);
        }
+       folio_batch_reinit(folios);
+}
+
+void free_unref_page_list(struct list_head *list)
+{
+       struct folio_batch fbatch;
+
+       folio_batch_init(&fbatch);
+       while (!list_empty(list)) {
+               struct folio *folio = list_first_entry(list, struct folio, lru);
+
+               list_del(&folio->lru);
+               if (folio_batch_add(&fbatch, folio) > 0)
+                       continue;
+               free_unref_folios(&fbatch);
+       }
+
+       if (fbatch.nr)
+               free_unref_folios(&fbatch);
 }
 
 /*