mm/mglru: improve struct lru_gen_mm_walk

Rename max_seq to seq in struct lru_gen_mm_walk to keep consistent with
struct lru_gen_mm_state.  Note that seq is not always up to date with
max_seq from lru_gen_folio.

No functional changes.

Link: https://lkml.kernel.org/r/20240214060538.3524462-5-kinseyho@google.com
Signed-off-by: Kinsey Ho <kinseyho@google.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Cc: Donet Tom <donettom@linux.vnet.ibm.com>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index a497f189d98818bcda37458746ebb2bded7826e4..633812a1d220241f49562ac91479e186f017b475 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -464,7 +464,7 @@ enum {
 #define NR_BLOOM_FILTERS       2
 
 struct lru_gen_mm_state {
-       /* set to max_seq after each iteration */
+       /* synced with max_seq after each iteration */
        unsigned long seq;
        /* where the current iteration continues after */
        struct list_head *head;
@@ -479,8 +479,8 @@ struct lru_gen_mm_state {
 struct lru_gen_mm_walk {
        /* the lruvec under reclaim */
        struct lruvec *lruvec;
-       /* unstable max_seq from lru_gen_folio */
-       unsigned long max_seq;
+       /* max_seq from lru_gen_folio: can be out of date */
+       unsigned long seq;
        /* the next address within an mm to scan */
        unsigned long next_addr;
        /* to batch promoted pages */

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 1bec586d854ea0eece104092cf1d4b37707bb32c..577e7f9be7c6caa77adf35a7f307275f8840fe85 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2888,7 +2888,7 @@ static void reset_mm_stats(struct lru_gen_mm_walk *walk, bool last)
 
        lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock);
 
-       hist = lru_hist_from_seq(walk->max_seq);
+       hist = lru_hist_from_seq(walk->seq);
 
        for (i = 0; i < NR_MM_STATS; i++) {
                WRITE_ONCE(mm_state->stats[hist][i],
@@ -2897,7 +2897,7 @@ static void reset_mm_stats(struct lru_gen_mm_walk *walk, bool last)
        }
 
        if (NR_HIST_GENS > 1 && last) {
-               hist = lru_hist_from_seq(walk->max_seq + 1);
+               hist = lru_hist_from_seq(walk->seq + 1);
 
                for (i = 0; i < NR_MM_STATS; i++)
                        WRITE_ONCE(mm_state->stats[hist][i], 0);
@@ -2926,9 +2926,9 @@ static bool iterate_mm_list(struct lru_gen_mm_walk *walk, struct mm_struct **ite
         */
        spin_lock(&mm_list->lock);
 
-       VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->max_seq);
+       VM_WARN_ON_ONCE(mm_state->seq + 1 < walk->seq);
 
-       if (walk->max_seq <= mm_state->seq)
+       if (walk->seq <= mm_state->seq)
                goto done;
 
        if (!mm_state->head)
@@ -2958,7 +2958,7 @@ done:
        spin_unlock(&mm_list->lock);
 
        if (mm && first)
-               reset_bloom_filter(mm_state, walk->max_seq + 1);
+               reset_bloom_filter(mm_state, walk->seq + 1);
 
        if (*iter)
                mmput_async(*iter);
@@ -2968,7 +2968,7 @@ done:
        return last;
 }
 
-static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq)
+static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long seq)
 {
        bool success = false;
        struct mem_cgroup *memcg = lruvec_memcg(lruvec);
@@ -2977,9 +2977,9 @@ static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long max_seq)
 
        spin_lock(&mm_list->lock);
 
-       VM_WARN_ON_ONCE(mm_state->seq + 1 < max_seq);
+       VM_WARN_ON_ONCE(mm_state->seq + 1 < seq);
 
-       if (max_seq > mm_state->seq) {
+       if (seq > mm_state->seq) {
                mm_state->head = NULL;
                mm_state->tail = NULL;
                WRITE_ONCE(mm_state->seq, mm_state->seq + 1);
@@ -3330,7 +3330,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end,
        struct lru_gen_mm_walk *walk = args->private;
        struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
        struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
-       int old_gen, new_gen = lru_gen_from_seq(walk->max_seq);
+       DEFINE_MAX_SEQ(walk->lruvec);
+       int old_gen, new_gen = lru_gen_from_seq(max_seq);
 
        pte = pte_offset_map_nolock(args->mm, pmd, start & PMD_MASK, &ptl);
        if (!pte)
@@ -3397,7 +3398,8 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area
        struct lru_gen_mm_walk *walk = args->private;
        struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
        struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
-       int old_gen, new_gen = lru_gen_from_seq(walk->max_seq);
+       DEFINE_MAX_SEQ(walk->lruvec);
+       int old_gen, new_gen = lru_gen_from_seq(max_seq);
 
        VM_WARN_ON_ONCE(pud_leaf(*pud));
 
@@ -3528,7 +3530,7 @@ restart:
                        walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);
                }
 
-               if (!walk->force_scan && !test_bloom_filter(mm_state, walk->max_seq, pmd + i))
+               if (!walk->force_scan && !test_bloom_filter(mm_state, walk->seq, pmd + i))
                        continue;
 
                walk->mm_stats[MM_NONLEAF_FOUND]++;
@@ -3539,7 +3541,7 @@ restart:
                walk->mm_stats[MM_NONLEAF_ADDED]++;
 
                /* carry over to the next generation */
-               update_bloom_filter(mm_state, walk->max_seq + 1, pmd + i);
+               update_bloom_filter(mm_state, walk->seq + 1, pmd + i);
        }
 
        walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);
@@ -3610,7 +3612,7 @@ static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
                err = -EBUSY;
 
                /* another thread might have called inc_max_seq() */
-               if (walk->max_seq != max_seq)
+               if (walk->seq != max_seq)
                        break;
 
                /* folio_update_gen() requires stable folio_memcg() */
@@ -3747,7 +3749,7 @@ next:
        return success;
 }
 
-static bool inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq,
                        bool can_swap, bool force_scan)
 {
        bool success;
@@ -3755,14 +3757,14 @@ static bool inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
        int type, zone;
        struct lru_gen_folio *lrugen = &lruvec->lrugen;
 restart:
-       if (max_seq < READ_ONCE(lrugen->max_seq))
+       if (seq < READ_ONCE(lrugen->max_seq))
                return false;
 
        spin_lock_irq(&lruvec->lru_lock);
 
        VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
 
-       success = max_seq == lrugen->max_seq;
+       success = seq == lrugen->max_seq;
        if (!success)
                goto unlock;
 
@@ -3815,7 +3817,7 @@ unlock:
        return success;
 }
 
-static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
+static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
                               bool can_swap, bool force_scan)
 {
        bool success;
@@ -3824,13 +3826,13 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
        struct lru_gen_folio *lrugen = &lruvec->lrugen;
        struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
 
-       VM_WARN_ON_ONCE(max_seq > READ_ONCE(lrugen->max_seq));
+       VM_WARN_ON_ONCE(seq > READ_ONCE(lrugen->max_seq));
 
        if (!mm_state)
-               return inc_max_seq(lruvec, max_seq, can_swap, force_scan);
+               return inc_max_seq(lruvec, seq, can_swap, force_scan);
 
        /* see the comment in iterate_mm_list() */
-       if (max_seq <= READ_ONCE(mm_state->seq))
+       if (seq <= READ_ONCE(mm_state->seq))
                return false;
 
        /*
@@ -3840,18 +3842,18 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
         * is less efficient, but it avoids bursty page faults.
         */
        if (!should_walk_mmu()) {
-               success = iterate_mm_list_nowalk(lruvec, max_seq);
+               success = iterate_mm_list_nowalk(lruvec, seq);
                goto done;
        }
 
        walk = set_mm_walk(NULL, true);
        if (!walk) {
-               success = iterate_mm_list_nowalk(lruvec, max_seq);
+               success = iterate_mm_list_nowalk(lruvec, seq);
                goto done;
        }
 
        walk->lruvec = lruvec;
-       walk->max_seq = max_seq;
+       walk->seq = seq;
        walk->can_swap = can_swap;
        walk->force_scan = force_scan;
 
@@ -3862,7 +3864,7 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long max_seq,
        } while (mm);
 done:
        if (success) {
-               success = inc_max_seq(lruvec, max_seq, can_swap, force_scan);
+               success = inc_max_seq(lruvec, seq, can_swap, force_scan);
                WARN_ON_ONCE(!success);
        }