mm/damon/schemes: add 'LRU_PRIO' DAMOS action

This commit adds a new DAMOS action called 'LRU_PRIO' for the physical
address space.  The action prioritizes pages in the memory regions of the
user-specified target access pattern on their LRU lists.  This is hence
supposed to be used for frequently accessed (hot) memory regions so that
hot pages could be more likely protected under memory pressure.
Internally, it simply calls 'mark_page_accessed()'.

Link: https://lkml.kernel.org/r/20220613192301.8817-5-sj@kernel.org
Signed-off-by: SeongJae Park <sj@kernel.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/damon.h b/include/linux/damon.h
index b9aae19fab3e9fe444b6337b8aa14e697449d121..4c64e03e94d82f084e9913f6d3ab17ba70f8cd25 100644 (file)
--- a/include/linux/damon.h
+++ b/include/linux/damon.h
@@ -86,6 +86,7 @@ struct damon_target {
  * @DAMOS_PAGEOUT:     Call ``madvise()`` for the region with MADV_PAGEOUT.
  * @DAMOS_HUGEPAGE:    Call ``madvise()`` for the region with MADV_HUGEPAGE.
  * @DAMOS_NOHUGEPAGE:  Call ``madvise()`` for the region with MADV_NOHUGEPAGE.
+ * @DAMOS_LRU_PRIO:    Prioritize the region on its LRU lists.
  * @DAMOS_STAT:                Do nothing but count the stat.
  * @NR_DAMOS_ACTIONS:  Total number of DAMOS actions
  */
@@ -95,6 +96,7 @@ enum damos_action {
        DAMOS_PAGEOUT,
        DAMOS_HUGEPAGE,
        DAMOS_NOHUGEPAGE,
+       DAMOS_LRU_PRIO,
        DAMOS_STAT,             /* Do nothing but only record the stat */
        NR_DAMOS_ACTIONS,
 };

diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c
index 10ef20b2003f5c340a4685e58fa81b2e66ffbb6c..b1335de200e77c6d6e34b91227f8c30945187373 100644 (file)
--- a/mm/damon/ops-common.c
+++ b/mm/damon/ops-common.c
@@ -130,3 +130,45 @@ int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
        /* Return coldness of the region */
        return DAMOS_MAX_SCORE - hotness;
 }
+
+int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
+                       struct damos *s)
+{
+       unsigned int max_nr_accesses;
+       int freq_subscore;
+       unsigned int age_in_sec;
+       int age_in_log, age_subscore;
+       unsigned int freq_weight = s->quota.weight_nr_accesses;
+       unsigned int age_weight = s->quota.weight_age;
+       int hotness;
+
+       max_nr_accesses = c->aggr_interval / c->sample_interval;
+       freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / max_nr_accesses;
+
+       age_in_sec = (unsigned long)r->age * c->aggr_interval / 1000000;
+       for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
+                       age_in_log++, age_in_sec >>= 1)
+               ;
+
+       /* If frequency is 0, higher age means it's colder */
+       if (freq_subscore == 0)
+               age_in_log *= -1;
+
+       /*
+        * Now age_in_log is in [-DAMON_MAX_AGE_IN_LOG, DAMON_MAX_AGE_IN_LOG].
+        * Scale it to be in [0, 100] and set it as age subscore.
+        */
+       age_in_log += DAMON_MAX_AGE_IN_LOG;
+       age_subscore = age_in_log * DAMON_MAX_SUBSCORE /
+               DAMON_MAX_AGE_IN_LOG / 2;
+
+       hotness = (freq_weight * freq_subscore + age_weight * age_subscore);
+       if (freq_weight + age_weight)
+               hotness /= freq_weight + age_weight;
+       /*
+        * Transform it to fit in [0, DAMOS_MAX_SCORE]
+        */
+       hotness = hotness * DAMOS_MAX_SCORE / DAMON_MAX_SUBSCORE;
+
+       return hotness;
+}

diff --git a/mm/damon/ops-common.h b/mm/damon/ops-common.h
index e790cb5f8fe05913ef868bd41743604c91d4aacf..52329ff361cd050850f000d513e8ea16276aa1d8 100644 (file)
--- a/mm/damon/ops-common.h
+++ b/mm/damon/ops-common.h
@@ -14,3 +14,5 @@ void damon_pmdp_mkold(pmd_t *pmd, struct mm_struct *mm, unsigned long addr);
 
 int damon_pageout_score(struct damon_ctx *c, struct damon_region *r,
                        struct damos *s);
+int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
+                       struct damos *s);

diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c
index 7bcd48066b4362b30329219c2ee366fc54aed7a8..f145b1d51e137d9951ef0a28dd1e442f5f8768df 100644 (file)
--- a/mm/damon/paddr.c
+++ b/mm/damon/paddr.c
@@ -233,6 +233,22 @@ static unsigned long damon_pa_pageout(struct damon_region *r)
        return applied * PAGE_SIZE;
 }
 
+static unsigned long damon_pa_mark_accessed(struct damon_region *r)
+{
+       unsigned long addr, applied = 0;
+
+       for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
+               struct page *page = damon_get_page(PHYS_PFN(addr));
+
+               if (!page)
+                       continue;
+               mark_page_accessed(page);
+               put_page(page);
+               applied++;
+       }
+       return applied * PAGE_SIZE;
+}
+
 static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,
                struct damon_target *t, struct damon_region *r,
                struct damos *scheme)
@@ -240,6 +256,8 @@ static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,
        switch (scheme->action) {
        case DAMOS_PAGEOUT:
                return damon_pa_pageout(r);
+       case DAMOS_LRU_PRIO:
+               return damon_pa_mark_accessed(r);
        default:
                break;
        }
@@ -253,6 +271,8 @@ static int damon_pa_scheme_score(struct damon_ctx *context,
        switch (scheme->action) {
        case DAMOS_PAGEOUT:
                return damon_pageout_score(context, r, scheme);
+       case DAMOS_LRU_PRIO:
+               return damon_hot_score(context, r, scheme);
        default:
                break;
        }

diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c
index c35809c6087ce7535718605ec14186d699535f73..86c69f980927d4e87c9d14513d51d3ef1e1dd9b0 100644 (file)
--- a/mm/damon/sysfs.c
+++ b/mm/damon/sysfs.c
@@ -762,6 +762,7 @@ static const char * const damon_sysfs_damos_action_strs[] = {
        "pageout",
        "hugepage",
        "nohugepage",
+       "lru_prio",
        "stat",
 };