mm: free folios directly in move_folios_to_lru()

The few folios which can't be moved to the LRU list (because their
refcount dropped to zero) used to be returned to the caller to dispose of.
Make this simpler to call by freeing the folios directly through
free_unref_folios().

Link: https://lkml.kernel.org/r/20240227174254.710559-13-willy@infradead.org
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 144879d5bb895fc518fff4529ca5ee186012a507..e3349b75f15bab37d1bf0561ae1773f1ba2b1143 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1781,7 +1781,6 @@ static bool too_many_isolated(struct pglist_data *pgdat, int file,
 
 /*
  * move_folios_to_lru() moves folios from private @list to appropriate LRU list.
- * On return, @list is reused as a list of folios to be freed by the caller.
  *
  * Returns the number of pages moved to the given lruvec.
  */
@@ -1789,8 +1788,9 @@ static unsigned int move_folios_to_lru(struct lruvec *lruvec,
                struct list_head *list)
 {
        int nr_pages, nr_moved = 0;
-       LIST_HEAD(folios_to_free);
+       struct folio_batch free_folios;
 
+       folio_batch_init(&free_folios);
        while (!list_empty(list)) {
                struct folio *folio = lru_to_folio(list);
 
@@ -1819,12 +1819,12 @@ static unsigned int move_folios_to_lru(struct lruvec *lruvec,
                if (unlikely(folio_put_testzero(folio))) {
                        __folio_clear_lru_flags(folio);
 
-                       if (unlikely(folio_test_large(folio))) {
+                       if (folio_batch_add(&free_folios, folio) == 0) {
                                spin_unlock_irq(&lruvec->lru_lock);
-                               destroy_large_folio(folio);
+                               mem_cgroup_uncharge_folios(&free_folios);
+                               free_unref_folios(&free_folios);
                                spin_lock_irq(&lruvec->lru_lock);
-                       } else
-                               list_add(&folio->lru, &folios_to_free);
+                       }
 
                        continue;
                }
@@ -1841,10 +1841,12 @@ static unsigned int move_folios_to_lru(struct lruvec *lruvec,
                        workingset_age_nonresident(lruvec, nr_pages);
        }
 
-       /*
-        * To save our caller's stack, now use input list for pages to free.
-        */
-       list_splice(&folios_to_free, list);
+       if (free_folios.nr) {
+               spin_unlock_irq(&lruvec->lru_lock);
+               mem_cgroup_uncharge_folios(&free_folios);
+               free_unref_folios(&free_folios);
+               spin_lock_irq(&lruvec->lru_lock);
+       }
 
        return nr_moved;
 }
@@ -1923,8 +1925,6 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
        spin_unlock_irq(&lruvec->lru_lock);
 
        lru_note_cost(lruvec, file, stat.nr_pageout, nr_scanned - nr_reclaimed);
-       mem_cgroup_uncharge_list(&folio_list);
-       free_unref_page_list(&folio_list);
 
        /*
         * If dirty folios are scanned that are not queued for IO, it
@@ -2065,8 +2065,6 @@ static void shrink_active_list(unsigned long nr_to_scan,
 
        nr_activate = move_folios_to_lru(lruvec, &l_active);
        nr_deactivate = move_folios_to_lru(lruvec, &l_inactive);
-       /* Keep all free folios in l_active list */
-       list_splice(&l_inactive, &l_active);
 
        __count_vm_events(PGDEACTIVATE, nr_deactivate);
        __count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
@@ -2076,8 +2074,6 @@ static void shrink_active_list(unsigned long nr_to_scan,
 
        if (nr_rotated)
                lru_note_cost(lruvec, file, 0, nr_rotated);
-       mem_cgroup_uncharge_list(&l_active);
-       free_unref_page_list(&l_active);
        trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate,
                        nr_deactivate, nr_rotated, sc->priority, file);
 }
@@ -4578,10 +4574,6 @@ retry:
 
        spin_unlock_irq(&lruvec->lru_lock);
 
-       mem_cgroup_uncharge_list(&list);
-       free_unref_page_list(&list);
-
-       INIT_LIST_HEAD(&list);
        list_splice_init(&clean, &list);
 
        if (!list_empty(&list)) {