mm: add statistics for PUD level pagetable

Recently, we found that cross-die access to pagetable pages on ARM64
machines can cause performance fluctuations in our business.  Currently,
there are no PMU events available to track this situation on our ARM64
machines, so accurate pagetable accounting can help to analyze this issue,
but now the PUD level pagetable accounting is missed.

So introduce pagetable_pud_ctor/dtor() to help to get accurate PUD
pagetable accounting, as well as converting the architectures which use
generic PUD pagetable allocation to add corresponding PUD pagetable
accounting.  Moreover this patch will mark the PUD level pagetable with
PG_table flag, which will help to do sanity validation in
unpoison_memory().

On my testing machine, I can see more pagetables statistics after the patch
with page-types tool:

Before patch:
        flags           page-count      MB  symbolic-flags                     long-symbolic-flags
0x0000000004000000           27326      106  __________________________g_________________       pgtable
After patch:
0x0000000004000000           27541      107  __________________________g_________________       pgtable

Link: https://lkml.kernel.org/r/876c71c03a7e69c17722a690e3225a4f7b172fb2.1695017383.git.baolin.wang@linux.alibaba.com
Signed-off-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Acked-by: Mike Rapoport (IBM) <rppt@kernel.org>
Acked-by: Vishal Moola (Oracle) <vishal.moola@gmail.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Bogendoerfer <tsbogend@alpha.franken.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index 2c29239d05c3e1decd3f2266187419ec2703888b..846c563689a81871ecff4736ac3c7c6f9e3831ff 100644 (file)
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -96,7 +96,10 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
                                  unsigned long addr)
 {
-       tlb_remove_ptdesc(tlb, virt_to_ptdesc(pudp));
+       struct ptdesc *ptdesc = virt_to_ptdesc(pudp);
+
+       pagetable_pud_dtor(ptdesc);
+       tlb_remove_ptdesc(tlb, ptdesc);
 }
 #endif
 

diff --git a/arch/loongarch/include/asm/pgalloc.h b/arch/loongarch/include/asm/pgalloc.h
index 79470f0b4f1d8dfbd715e4d1dc2e711949175b30..4e2d6b7ca2eebf5ba14bfa46ae2699e37716d210 100644 (file)
--- a/arch/loongarch/include/asm/pgalloc.h
+++ b/arch/loongarch/include/asm/pgalloc.h
@@ -84,6 +84,7 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address)
 
        if (!ptdesc)
                return NULL;
+       pagetable_pud_ctor(ptdesc);
        pud = ptdesc_address(ptdesc);
 
        pud_init(pud);

diff --git a/arch/mips/include/asm/pgalloc.h b/arch/mips/include/asm/pgalloc.h
index 40e40a7eb94af582c690f72ec52f99efe2704df3..f4440edcd8fe2eec7529362c004143ad6fe05c91 100644 (file)
--- a/arch/mips/include/asm/pgalloc.h
+++ b/arch/mips/include/asm/pgalloc.h
@@ -95,6 +95,7 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long address)
 
        if (!ptdesc)
                return NULL;
+       pagetable_pud_ctor(ptdesc);
        pud = ptdesc_address(ptdesc);
 
        pud_init(pud);

diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 9deadf517f14a94c75f1509ba40678cc544b8a5b..0cbc1b8e8e3d10185e4e63ff3fab4e0057417f99 100644 (file)
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -76,6 +76,9 @@ void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
 #if CONFIG_PGTABLE_LEVELS > 3
 void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
 {
+       struct ptdesc *ptdesc = virt_to_ptdesc(pud);
+
+       pagetable_pud_dtor(ptdesc);
        paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
        paravirt_tlb_remove_table(tlb, virt_to_page(pud));
 }

diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h
index c75d4a753849398f3c0bae8d60104c7ba3849bd0..879e5f8aa5e92a1b419f6bc910c263a6f0299812 100644 (file)
--- a/include/asm-generic/pgalloc.h
+++ b/include/asm-generic/pgalloc.h
@@ -169,6 +169,8 @@ static inline pud_t *__pud_alloc_one(struct mm_struct *mm, unsigned long addr)
        ptdesc = pagetable_alloc(gfp, 0);
        if (!ptdesc)
                return NULL;
+
+       pagetable_pud_ctor(ptdesc);
        return ptdesc_address(ptdesc);
 }
 
@@ -190,8 +192,11 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
 
 static inline void __pud_free(struct mm_struct *mm, pud_t *pud)
 {
+       struct ptdesc *ptdesc = virt_to_ptdesc(pud);
+
        BUG_ON((unsigned long)pud & (PAGE_SIZE-1));
-       pagetable_free(virt_to_ptdesc(pud));
+       pagetable_pud_dtor(ptdesc);
+       pagetable_free(ptdesc);
 }
 
 #ifndef __HAVE_ARCH_PUD_FREE

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 31dc25d3f6b5a94df4ddb7f798a83ac282d784e4..126b54b4544235d11a5eaa48bab680419e7da0c1 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -3049,6 +3049,22 @@ static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud)
        return ptl;
 }
 
+static inline void pagetable_pud_ctor(struct ptdesc *ptdesc)
+{
+       struct folio *folio = ptdesc_folio(ptdesc);
+
+       __folio_set_pgtable(folio);
+       lruvec_stat_add_folio(folio, NR_PAGETABLE);
+}
+
+static inline void pagetable_pud_dtor(struct ptdesc *ptdesc)
+{
+       struct folio *folio = ptdesc_folio(ptdesc);
+
+       __folio_clear_pgtable(folio);
+       lruvec_stat_sub_folio(folio, NR_PAGETABLE);
+}
+
 extern void __init pagecache_init(void);
 extern void free_initmem(void);