thp: add thp_get_unmapped_area_vmflags()

When memory is being placed, mmap() will take care to respect the guard
gaps of certain types of memory (VM_SHADOWSTACK, VM_GROWSUP and
VM_GROWSDOWN).  In order to ensure guard gaps between mappings, mmap()
needs to consider two things:

 1. That the new mapping isn't placed in an any existing mappings guard
    gaps.
 2. That the new mapping isn't placed such that any existing mappings
    are not in *its* guard gaps.

The longstanding behavior of mmap() is to ensure 1, but not take any care
around 2.  So for example, if there is a PAGE_SIZE free area, and a mmap()
with a PAGE_SIZE size, and a type that has a guard gap is being placed,
mmap() may place the shadow stack in the PAGE_SIZE free area.  Then the
mapping that is supposed to have a guard gap will not have a gap to the
adjacent VMA.

Add a THP implementations of the vm_flags variant of get_unmapped_area().
Future changes will call this from mmap.c in the do_mmap() path to allow
shadow stacks to be placed with consideration taken for the start guard
gap.  Shadow stack memory is always private and anonymous and so special
guard gap logic is not needed in a lot of caseis, but it can be mapped by
THP, so needs to be handled.

Link: https://lkml.kernel.org/r/20240326021656.202649-7-rick.p.edgecombe@intel.com
Signed-off-by: Rick Edgecombe <rick.p.edgecombe@intel.com>
Reviewed-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Aneesh Kumar K.V <aneesh.kumar@kernel.org>
Cc: Borislav Petkov (AMD) <bp@alien8.de>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Deepak Gupta <debug@rivosinc.com>
Cc: Guo Ren <guoren@kernel.org>
Cc: Helge Deller <deller@gmx.de>
Cc: H. Peter Anvin (Intel) <hpa@zytor.com>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Liam R. Howlett <Liam.Howlett@oracle.com>
Cc: Mark Brown <broonie@kernel.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Naveen N. Rao <naveen.n.rao@linux.ibm.com>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index ab26d9e65ec3be430cd7fd5d796e9ac4b7772f79..e896ca4760f673c46d3310201724545cc973d4ec 100644 (file)
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -270,6 +270,9 @@ unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma,
 
 unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
                unsigned long len, unsigned long pgoff, unsigned long flags);
+unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
+               unsigned long len, unsigned long pgoff, unsigned long flags,
+               vm_flags_t vm_flags);
 
 bool can_split_folio(struct folio *folio, int *pextra_pins);
 int split_huge_page_to_list_to_order(struct page *page, struct list_head *list,
@@ -413,6 +416,14 @@ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma,
 
 #define thp_get_unmapped_area  NULL
 
+static inline unsigned long
+thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
+                             unsigned long len, unsigned long pgoff,
+                             unsigned long flags, vm_flags_t vm_flags)
+{
+       return 0;
+}
+
 static inline bool
 can_split_folio(struct folio *folio, int *pextra_pins)
 {

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 7fdb2275a1e0403a3ddfec1b34c265a9472397aa..b757d9be310c8aee654b214a634ae3b0c023493c 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -800,7 +800,8 @@ static inline bool is_transparent_hugepage(const struct folio *folio)
 
 static unsigned long __thp_get_unmapped_area(struct file *filp,
                unsigned long addr, unsigned long len,
-               loff_t off, unsigned long flags, unsigned long size)
+               loff_t off, unsigned long flags, unsigned long size,
+               vm_flags_t vm_flags)
 {
        loff_t off_end = off + len;
        loff_t off_align = round_up(off, size);
@@ -816,8 +817,8 @@ static unsigned long __thp_get_unmapped_area(struct file *filp,
        if (len_pad < len || (off + len_pad) < off)
                return 0;
 
-       ret = mm_get_unmapped_area(current->mm, filp, addr, len_pad,
-                                  off >> PAGE_SHIFT, flags);
+       ret = mm_get_unmapped_area_vmflags(current->mm, filp, addr, len_pad,
+                                          off >> PAGE_SHIFT, flags, vm_flags);
 
        /*
         * The failure might be due to length padding. The caller will retry
@@ -842,17 +843,25 @@ static unsigned long __thp_get_unmapped_area(struct file *filp,
        return ret;
 }
 
-unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
-               unsigned long len, unsigned long pgoff, unsigned long flags)
+unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr,
+               unsigned long len, unsigned long pgoff, unsigned long flags,
+               vm_flags_t vm_flags)
 {
        unsigned long ret;
        loff_t off = (loff_t)pgoff << PAGE_SHIFT;
 
-       ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE);
+       ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags);
        if (ret)
                return ret;
 
-       return mm_get_unmapped_area(current->mm, filp, addr, len, pgoff, flags);
+       return mm_get_unmapped_area_vmflags(current->mm, filp, addr, len, pgoff, flags,
+                                           vm_flags);
+}
+
+unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
+               unsigned long len, unsigned long pgoff, unsigned long flags)
+{
+       return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0);
 }
 EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
 

diff --git a/mm/mmap.c b/mm/mmap.c
index a37c8dd29d64f7d83867e6b67b8c483656282673..0ef1191e3be5b04d133cc7a4b09333b40f559d22 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1860,20 +1860,22 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
                 * so use shmem's get_unmapped_area in case it can be huge.
                 */
                get_area = shmem_get_unmapped_area;
-       } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
-               /* Ensures that larger anonymous mappings are THP aligned. */
-               get_area = thp_get_unmapped_area;
        }
 
        /* Always treat pgoff as zero for anonymous memory. */
        if (!file)
                pgoff = 0;
 
-       if (get_area)
+       if (get_area) {
                addr = get_area(file, addr, len, pgoff, flags);
-       else
+       } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
+               /* Ensures that larger anonymous mappings are THP aligned. */
+               addr = thp_get_unmapped_area_vmflags(file, addr, len,
+                                                    pgoff, flags, vm_flags);
+       } else {
                addr = mm_get_unmapped_area_vmflags(current->mm, file, addr, len,
                                                    pgoff, flags, vm_flags);
+       }
        if (IS_ERR_VALUE(addr))
                return addr;